vpnservice/bgpmanager/bgpmanager-impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright (c) 2015 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netvirt.bgpmanager;
  10
  11 import com.google.common.base.Optional;
  12
  13 import java.io.*;
  14 import java.util.*;
  15 import java.lang.reflect.*;
  16 import java.net.InetAddress;
  17 import java.net.NetworkInterface;
  18 import java.util.concurrent.*;
  19 import java.util.concurrent.atomic.AtomicBoolean;
  20
  21 import org.opendaylight.netvirt.bgpmanager.api.RouteOrigin;
  22 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  23 import org.opendaylight.netvirt.bgpmanager.thrift.gen.*;
  24 import org.opendaylight.netvirt.bgpmanager.thrift.client.*;
  25 import org.opendaylight.netvirt.bgpmanager.thrift.server.*;
  26 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  27 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  28 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  29 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipChange;
  30 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipListener;
  31 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipService;
  32 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  33 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  34 import org.opendaylight.genius.utils.clustering.EntityOwnerUtils;
  35 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.*;
  36 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.*;
  37 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.*;
  38 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.*;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  42 import org.opendaylight.yangtools.concepts.ListenerRegistration;
  43 import org.opendaylight.yangtools.yang.binding.DataObject;
  44 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  45 import org.osgi.framework.Bundle;
  46 import org.osgi.framework.BundleContext;
  47 import org.osgi.framework.FrameworkUtil;
  48 import org.slf4j.Logger;
  49 import org.slf4j.LoggerFactory;
  50
  51 public class BgpConfigurationManager {
  52     private static final Logger LOG =
  53             LoggerFactory.getLogger(BgpConfigurationManager.class);
  54     private static DataBroker broker;
  55     private static FibDSWriter fib;
  56     private static Bgp config;
  57     private static BgpRouter bgpRouter;
  58     private static BgpThriftService updateServer;
  59
  60     private static final String DEF_LOGFILE = "/var/log/bgp_debug.log";
  61     private static final String DEF_LOGLEVEL = "errors";
  62     private static final String UPDATE_PORT = "bgp.thrift.service.port";
  63     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
  64     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
  65     private static final String DEF_UPORT = "6644";
  66     private static final String DEF_CHOST = "127.0.0.1";
  67     private static final String DEF_CPORT = "7644";
  68     private static final String SDNC_BGP_MIP = "sdnc_bgp_mip";
  69     private static final String CLUSTER_CONF_FILE = "/cluster/etc/cluster.conf";
  70     private static final Timer ipActivationCheckTimer = new Timer();
  71     private static final int STALE_FIB_WAIT = 60;
  72     private static final int RESTART_DEFAULT_GR = 90;
  73     private long StaleStartTime = 0;
  74     private long StaleEndTime = 0;
  75     private long CfgReplayStartTime = 0;
  76     private long CfgReplayEndTime = 0;
  77     private long StaleCleanupTime = 0;
  78
  79     public long getStaleCleanupTime() {
  80         return StaleCleanupTime;
  81     }
  82
  83     public void setStaleCleanupTime(long staleCleanupTime) {
  84         StaleCleanupTime = staleCleanupTime;
  85     }
  86
  87     public long getCfgReplayEndTime() {
  88         return CfgReplayEndTime;
  89     }
  90
  91     public void setCfgReplayEndTime(long cfgReplayEndTime) {
  92         CfgReplayEndTime = cfgReplayEndTime;
  93     }
  94
  95     public long getCfgReplayStartTime() {
  96         return CfgReplayStartTime;
  97     }
  98
  99     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 100         CfgReplayStartTime = cfgReplayStartTime;
 101     }
 102
 103     public long getStaleEndTime() {
 104         return StaleEndTime;
 105     }
 106
 107     public void setStaleEndTime(long staleEndTime) {
 108         StaleEndTime = staleEndTime;
 109     }
 110
 111     public long getStaleStartTime() {
 112         return StaleStartTime;
 113     }
 114
 115     public void setStaleStartTime(long staleStartTime) {
 116         StaleStartTime = staleStartTime;
 117     }
 118
 119
 120     // to have stale FIB map (RD, Prefix)
 121     //  number of seconds wait for route sync-up between ODL and BGP.
 122     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 360;
 123
 124     static String odlThriftIp = "127.0.0.1";
 125     private static String cHostStartup;
 126     private static String cPortStartup;
 127     private static CountDownLatch initer = new CountDownLatch(1);
 128     //static IITMProvider itmProvider;
 129     public static BgpManager bgpManager;
 130     //map<rd, map<prefix/len, nexthop/label>>
 131     private static Map<String, Map<String, String>> staledFibEntriesMap = new ConcurrentHashMap<>();
 132
 133     static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 134     static final String BGP_ENTITY_NAME = "bgp";
 135
 136     static int totalStaledCount = 0;
 137     static int totalCleared = 0;
 138
 139     private static final Class[] reactors =
 140             {
 141                     ConfigServerReactor.class, AsIdReactor.class,
 142                     GracefulRestartReactor.class, LoggingReactor.class,
 143                     NeighborsReactor.class, UpdateSourceReactor.class,
 144                     EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 145                     NetworksReactor.class, VrfsReactor.class, BgpReactor.class
 146             };
 147
 148     private ListenerRegistration<DataChangeListener>[] registrations;
 149     EntityOwnershipService entityOwnershipService;
 150
 151     private Object createListener(Class<?> cls) {
 152         Constructor<?> ctor;
 153         Object obj = null;
 154
 155         try {
 156             ctor = cls.getConstructor(BgpConfigurationManager.class);
 157             obj = ctor.newInstance(this);
 158         } catch (Exception e) {
 159             LOG.error("Failed to create listener object", e);
 160         }
 161         return obj;
 162     }
 163
 164     private void registerCallbacks() {
 165         String emsg = "Failed to register listener";
 166         registrations = new ListenerRegistration[reactors.length];
 167         InstanceIdentifier<?> iid = InstanceIdentifier.create(Bgp.class);
 168         for (Class reactor : reactors) {
 169             Object obj = createListener(reactor);
 170             String dclName = obj.getClass().getName();
 171             try {
 172                 AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 173                 dcl.registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 174             } catch (Exception e) {
 175                 LOG.error(emsg, e);
 176                 throw new IllegalStateException(emsg + " " + dclName, e);
 177             }
 178         }
 179     }
 180
 181     public void close() {
 182         if (updateServer != null) {
 183             updateServer.stop();
 184         }
 185     }
 186
 187     private boolean configExists() throws InterruptedException, ExecutionException, TimeoutException {
 188         InstanceIdentifier.InstanceIdentifierBuilder<Bgp> iib =
 189                 InstanceIdentifier.builder(Bgp.class);
 190         InstanceIdentifier<Bgp> iid = iib.build();
 191         Optional<Bgp> b = BgpUtil.read(broker,
 192                 LogicalDatastoreType.CONFIGURATION, iid);
 193         return b.isPresent();
 194     }
 195
 196     private String getProperty(String var, String def) {
 197         Bundle b = FrameworkUtil.getBundle(BgpManager.class);
 198         if (b == null) {
 199             return def;
 200         }
 201         BundleContext context = b.getBundleContext();
 202         if (context == null) {
 203             return def;
 204         }
 205         String s = context.getProperty(var);
 206         return (s == null ? def : s);
 207     }
 208
 209     static BgpConfigurationManager bgpConfigurationManager;
 210
 211     public BgpConfigurationManager(BgpManager bgpMgr) throws InterruptedException, ExecutionException, TimeoutException {
 212         broker = bgpMgr.getBroker();
 213         fib = bgpMgr.getFibWriter();
 214
 215         bgpManager = bgpMgr;
 216         bgpRouter = BgpRouter.getInstance();
 217         String uPort = getProperty(UPDATE_PORT, DEF_UPORT);
 218         cHostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 219         cPortStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 220         VtyshCli.setHostAddr(cHostStartup);
 221         ClearBgpCli.setHostAddr(cHostStartup);
 222         LOG.info("UpdateServer at localhost:" + uPort + " ConfigServer at "
 223                 + cHostStartup + ":" + cPortStartup);
 224         updateServer = new BgpThriftService(Integer.parseInt(uPort), bgpMgr);
 225         updateServer.start();
 226         readOdlThriftIpForBgpCommunication();
 227         registerCallbacks();
 228
 229         LOG.info("BGP Configuration manager initialized");
 230         initer.countDown();
 231
 232         bgpConfigurationManager = this;
 233     }
 234
 235     boolean ignoreClusterDcnEventForFollower() {
 236         return !EntityOwnerUtils.amIEntityOwner(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME);
 237     }
 238
 239     public Bgp get() {
 240         config = bgpManager.getConfig();
 241         return config;
 242     }
 243
 244     public void setEntityOwnershipService(EntityOwnershipService entityOwnershipService) {
 245         this.entityOwnershipService = entityOwnershipService;
 246         try {
 247             EntityOwnerUtils.registerEntityCandidateForOwnerShip(entityOwnershipService,
 248                     BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME, new EntityOwnershipListener() {
 249                 @Override
 250                 public void ownershipChanged(EntityOwnershipChange ownershipChange) {
 251                     LOG.trace("entity owner change event fired");
 252                     if (ownershipChange.hasOwner() && ownershipChange.isOwner()) {
 253                         LOG.trace("This PL is the Owner");
 254                         activateMIP();
 255                         bgpRestarted();
 256                     } else {
 257                         LOG.error("Not owner: hasOwner: {}, isOwner: {}",ownershipChange.hasOwner(),
 258                                 ownershipChange.isOwner() );
 259                     }
 260                 }
 261             });
 262         } catch (Exception e) {
 263             LOG.error("failed to register bgp entity", e);
 264         }
 265     }
 266
 267     public EntityOwnershipService getEntityOwnershipService() {
 268         return entityOwnershipService;
 269     }
 270
 271     private static final String addWarn =
 272             "Config store updated; undo with Delete if needed.";
 273     private static final String delWarn =
 274             "Config store updated; undo with Add if needed.";
 275     private static final String updWarn =
 276             "Update operation not supported; Config store updated;"
 277                     + " restore with another Update if needed.";
 278
 279     public class ConfigServerReactor
 280             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 281             implements AutoCloseable, ClusteredDataTreeChangeListener <ConfigServer> {
 282         private static final String yangObj = "config-server ";
 283
 284         public ConfigServerReactor() {
 285             super(ConfigServer.class, ConfigServerReactor.class);
 286         }
 287
 288         protected synchronized void
 289         add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 290             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 291             if (ignoreClusterDcnEventForFollower()) {
 292                 return;
 293             }
 294
 295             try {
 296                 initer.await();
 297             } catch (Exception e) {
 298             }
 299             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 300             synchronized (BgpConfigurationManager.this) {
 301                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 302                         val.getPort().intValue());
 303                 if (!res) {
 304                     LOG.error(yangObj + "Add failed; " + addWarn);
 305                 }
 306             }
 307         }
 308
 309         @Override
 310         protected ConfigServerReactor getDataTreeChangeListener() {
 311             return ConfigServerReactor.this;
 312         }
 313
 314         @Override
 315         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 316             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 317         }
 318
 319         protected synchronized void
 320         remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 321             LOG.trace("received bgp disconnect");
 322             if (ignoreClusterDcnEventForFollower()) {
 323                 return;
 324             }
 325             synchronized (BgpConfigurationManager.this) {
 326                 bgpRouter.disconnect();
 327             }
 328         }
 329
 330         protected void update(InstanceIdentifier<ConfigServer> iid,
 331                               ConfigServer oldval, ConfigServer newval) {
 332             LOG.trace("received bgp Connection update");
 333             if (ignoreClusterDcnEventForFollower()) {
 334                 return;
 335             }
 336             LOG.error(yangObj + updWarn);
 337         }
 338
 339         @Override
 340         public void close() {
 341             try {
 342                 super.close();
 343             } catch (Exception e) {
 344                 e.printStackTrace();
 345             }
 346         }
 347     }
 348
 349     private BgpRouter getClient(String yangObj) {
 350         if (bgpRouter == null) {
 351             LOG.warn(yangObj + ": configuration received when BGP is inactive");
 352         }
 353         return bgpRouter;
 354     }
 355
 356     public class AsIdReactor
 357             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 358             implements AutoCloseable, ClusteredDataTreeChangeListener<AsId> {
 359
 360         private static final String yangObj = "as-id ";
 361
 362         public AsIdReactor() {
 363             super(AsId.class, AsIdReactor.class);
 364         }
 365
 366         protected synchronized void
 367         add(InstanceIdentifier<AsId> iid, AsId val) {
 368             LOG.error("received bgp add asid");
 369             if (ignoreClusterDcnEventForFollower()) {
 370                 return;
 371             }
 372             LOG.debug("received add router config asNum {}", val.getLocalAs().intValue());
 373             synchronized (BgpConfigurationManager.this) {
 374                 BgpRouter br = getClient(yangObj);
 375                 if (br == null) {
 376                     LOG.error("no bgp router client found exiting asid add");
 377                     return;
 378                 }
 379                 int asNum = val.getLocalAs().intValue();
 380                 Ipv4Address routerId = val.getRouterId();
 381                 Boolean afb = val.isAnnounceFbit();
 382                 String rid = (routerId == null) ? "" : routerId.getValue();
 383                 int stalepathTime = (int) getStalePathtime(RESTART_DEFAULT_GR, val);
 384                 boolean announceFbit = (afb == null) ? false : afb.booleanValue();
 385                 try {
 386                     br.startBgp(asNum, rid, stalepathTime, announceFbit);
 387                     if (bgpManager.getBgpCounters() == null) {
 388                         bgpManager.startBgpCountersTask();
 389                     }
 390                 } catch (BgpRouterException bre) {
 391                     if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
 392                         LOG.error(yangObj + "Add requested when BGP is already active");
 393                     } else {
 394                         LOG.error(yangObj + "Add received exception: \""
 395                                 + bre + "\"; " + addWarn);
 396                     }
 397                 } catch (Exception e) {
 398                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; " + addWarn);
 399                 }
 400             }
 401         }
 402
 403         @Override
 404         protected AsIdReactor getDataTreeChangeListener() {
 405             return AsIdReactor.this;
 406         }
 407
 408         @Override
 409         protected InstanceIdentifier<AsId> getWildCardPath() {
 410             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 411         }
 412
 413         protected synchronized void
 414         remove(InstanceIdentifier<AsId> iid, AsId val) {
 415             LOG.error("received delete router config asNum {}", val.getLocalAs().intValue());
 416             if (ignoreClusterDcnEventForFollower()) {
 417                 return;
 418             }
 419             synchronized (BgpConfigurationManager.this) {
 420                 BgpRouter br = getClient(yangObj);
 421                 if (br == null) {
 422                     return;
 423                 }
 424                 int asNum = val.getLocalAs().intValue();
 425                 try {
 426                     br.stopBgp(asNum);
 427                 } catch (Exception e) {
 428                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; " + delWarn);
 429                 }
 430                 if (bgpManager.getBgpCounters() != null) {
 431                     bgpManager.stopBgpCountersTask();
 432                 }
 433             }
 434         }
 435
 436         protected void update(InstanceIdentifier<AsId> iid,
 437                               AsId oldval, AsId newval) {
 438             if (ignoreClusterDcnEventForFollower()) {
 439                 return;
 440             }
 441             LOG.error(yangObj + updWarn);
 442         }
 443
 444         @Override
 445         public void close() {
 446             try {
 447                 super.close();
 448             } catch (Exception e) {
 449                 e.printStackTrace();
 450             }
 451         }
 452     }
 453
 454     public class GracefulRestartReactor
 455             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 456             implements AutoCloseable, ClusteredDataTreeChangeListener<GracefulRestart> {
 457
 458         private static final String yangObj = "graceful-restart ";
 459
 460         public GracefulRestartReactor() {
 461             super(GracefulRestart.class, GracefulRestartReactor.class);
 462         }
 463
 464         protected synchronized void
 465         add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 466             if (ignoreClusterDcnEventForFollower()) {
 467                 return;
 468             }
 469             synchronized (BgpConfigurationManager.this) {
 470                 BgpRouter br = getClient(yangObj);
 471                 if (br == null) {
 472                     return;
 473                 }
 474                 try {
 475                     br.addGracefulRestart(val.getStalepathTime().intValue());
 476                 } catch (Exception e) {
 477                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; " + addWarn);
 478                 }
 479             }
 480         }
 481
 482         @Override
 483         protected GracefulRestartReactor getDataTreeChangeListener() {
 484             return GracefulRestartReactor.this;
 485         }
 486
 487         @Override
 488         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 489             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 490         }
 491
 492         protected synchronized void
 493         remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 494             if (ignoreClusterDcnEventForFollower()) {
 495                 return;
 496             }
 497             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 498             synchronized (BgpConfigurationManager.this) {
 499                 BgpRouter br = getClient(yangObj);
 500                 if (br == null) {
 501                     return;
 502                 }
 503                 try {
 504                     br.delGracefulRestart();
 505                 } catch (Exception e) {
 506                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 507                             + delWarn);
 508                 }
 509             }
 510         }
 511
 512         protected void update(InstanceIdentifier<GracefulRestart> iid,
 513                               GracefulRestart oldval, GracefulRestart newval) {
 514             if (ignoreClusterDcnEventForFollower()) {
 515                 return;
 516             }
 517             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 518             synchronized (BgpConfigurationManager.this) {
 519                 BgpRouter br = getClient(yangObj);
 520                 if (br == null) {
 521                     return;
 522                 }
 523                 try {
 524                     br.addGracefulRestart(newval.getStalepathTime().intValue());
 525                 } catch (Exception e) {
 526                     LOG.error(yangObj + "update received exception: \"" + e + "\"; " + addWarn);
 527                 }
 528             }
 529         }
 530
 531         @Override
 532         public void close() {
 533             try {
 534                 super.close();
 535             } catch (Exception e) {
 536                 e.printStackTrace();
 537             }
 538         }
 539     }
 540
 541     public class LoggingReactor
 542             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 543             implements AutoCloseable, ClusteredDataTreeChangeListener<Logging> {
 544
 545         private static final String yangObj = "logging ";
 546
 547         public LoggingReactor() {
 548             super(Logging.class, LoggingReactor.class);
 549         }
 550
 551         protected synchronized void
 552         add(InstanceIdentifier<Logging> iid, Logging val) {
 553             if (ignoreClusterDcnEventForFollower()) {
 554                 return;
 555             }
 556             synchronized (BgpConfigurationManager.this) {
 557                 BgpRouter br = getClient(yangObj);
 558                 if (br == null) {
 559                     return;
 560                 }
 561                 try {
 562                     br.setLogging(val.getFile(), val.getLevel());
 563                 } catch (Exception e) {
 564                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 565                             + addWarn);
 566                 }
 567             }
 568         }
 569
 570         @Override
 571         protected LoggingReactor getDataTreeChangeListener() {
 572             return LoggingReactor.this;
 573         }
 574
 575         @Override
 576         protected InstanceIdentifier<Logging> getWildCardPath() {
 577             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 578         }
 579
 580         protected synchronized void
 581         remove(InstanceIdentifier<Logging> iid, Logging val) {
 582             if (ignoreClusterDcnEventForFollower()) {
 583                 return;
 584             }
 585             LOG.debug("received remove Logging config val {}", val.getLevel());
 586             synchronized (BgpConfigurationManager.this) {
 587                 BgpRouter br = getClient(yangObj);
 588                 if (br == null) {
 589                     return;
 590                 }
 591                 try {
 592                     br.setLogging(DEF_LOGFILE, DEF_LOGLEVEL);
 593                 } catch (Exception e) {
 594                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 595                             + delWarn);
 596                 }
 597             }
 598         }
 599
 600         protected void update(InstanceIdentifier<Logging> iid,
 601                               Logging oldval, Logging newval) {
 602             if (ignoreClusterDcnEventForFollower()) {
 603                 return;
 604             }
 605             synchronized (BgpConfigurationManager.this) {
 606                 BgpRouter br = getClient(yangObj);
 607                 if (br == null) {
 608                     return;
 609                 }
 610                 try {
 611                     br.setLogging(newval.getFile(), newval.getLevel());
 612                 } catch (Exception e) {
 613                     LOG.error(yangObj + "newval received exception: \"" + e + "\"; "
 614                             + addWarn);
 615                 }
 616             }
 617         }
 618
 619         @Override
 620         public void close() {
 621             try {
 622                 super.close();
 623             } catch (Exception e) {
 624                 e.printStackTrace();
 625             }
 626         }
 627     }
 628
 629     public class NeighborsReactor
 630             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 631             implements AutoCloseable, ClusteredDataTreeChangeListener<Neighbors> {
 632
 633         private static final String yangObj = "neighbors ";
 634
 635         public NeighborsReactor() {
 636             super(Neighbors.class, NeighborsReactor.class);
 637         }
 638
 639         protected synchronized void
 640         add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 641             if (ignoreClusterDcnEventForFollower()) {
 642                 return;
 643             }
 644             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 645             synchronized (BgpConfigurationManager.this) {
 646                 BgpRouter br = getClient(yangObj);
 647                 if (br == null) {
 648                     return;
 649                 }
 650                 String peerIp = val.getAddress().getValue();
 651                 int as = val.getRemoteAs().intValue();
 652                 try {
 653                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 654                     br.addNeighbor(peerIp, as);
 655
 656                 } catch (Exception e) {
 657                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 658                             + addWarn);
 659                 }
 660             }
 661         }
 662
 663         @Override
 664         protected NeighborsReactor getDataTreeChangeListener() {
 665             return NeighborsReactor.this;
 666         }
 667
 668         @Override
 669         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 670             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class);
 671         }
 672
 673         protected synchronized void
 674         remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 675             if (ignoreClusterDcnEventForFollower()) {
 676                 return;
 677             }
 678             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 679             synchronized (BgpConfigurationManager.this) {
 680                 BgpRouter br = getClient(yangObj);
 681                 if (br == null) {
 682                     return;
 683                 }
 684                 String peerIp = val.getAddress().getValue();
 685                 try {
 686                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 687                     br.delNeighbor(peerIp);
 688                 } catch (Exception e) {
 689                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 690                             + delWarn);
 691                 }
 692             }
 693         }
 694
 695         protected void update(InstanceIdentifier<Neighbors> iid,
 696                               Neighbors oldval, Neighbors newval) {
 697             if (ignoreClusterDcnEventForFollower()) {
 698                 return;
 699             }
 700             //purposefully nothing to do.
 701         }
 702
 703         @Override
 704         public void close() {
 705             try {
 706                 super.close();
 707             } catch (Exception e) {
 708                 e.printStackTrace();
 709             }
 710         }
 711     }
 712
 713     public class EbgpMultihopReactor
 714             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 715             implements AutoCloseable, ClusteredDataTreeChangeListener<EbgpMultihop> {
 716
 717         private static final String yangObj = "ebgp-multihop ";
 718
 719         public EbgpMultihopReactor() {
 720             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 721         }
 722
 723         protected synchronized void
 724         add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 725             if (ignoreClusterDcnEventForFollower()) {
 726                 return;
 727             }
 728             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 729             synchronized (BgpConfigurationManager.this) {
 730                 BgpRouter br = getClient(yangObj);
 731                 if (br == null) {
 732                     return;
 733                 }
 734                 String peerIp = val.getPeerIp().getValue();
 735                 try {
 736                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 737                 } catch (Exception e) {
 738                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 739                             + addWarn);
 740                 }
 741             }
 742         }
 743
 744         @Override
 745         protected EbgpMultihopReactor getDataTreeChangeListener() {
 746             return EbgpMultihopReactor.this;
 747         }
 748
 749         @Override
 750         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
 751             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(EbgpMultihop.class);
 752         }
 753
 754         protected synchronized void
 755         remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 756             if (ignoreClusterDcnEventForFollower()) {
 757                 return;
 758             }
 759             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 760             synchronized (BgpConfigurationManager.this) {
 761                 BgpRouter br = getClient(yangObj);
 762                 if (br == null) {
 763                     return;
 764                 }
 765                 String peerIp = val.getPeerIp().getValue();
 766                 try {
 767                     br.delEbgpMultihop(peerIp);
 768                 } catch (Exception e) {
 769                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 770                             + delWarn);
 771                 }
 772             }
 773         }
 774
 775         protected void update(InstanceIdentifier<EbgpMultihop> iid,
 776                               EbgpMultihop oldval, EbgpMultihop newval) {
 777             if (ignoreClusterDcnEventForFollower()) {
 778                 return;
 779             }
 780             LOG.error(yangObj + updWarn);
 781         }
 782
 783         @Override
 784         public void close() {
 785             try {
 786                 super.close();
 787             } catch (Exception e) {
 788                 e.printStackTrace();
 789             }
 790         }
 791     }
 792
 793     public class UpdateSourceReactor
 794             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
 795             implements AutoCloseable, ClusteredDataTreeChangeListener<UpdateSource> {
 796
 797         private static final String yangObj = "update-source ";
 798
 799         public UpdateSourceReactor() {
 800             super(UpdateSource.class, UpdateSourceReactor.class);
 801         }
 802
 803         protected synchronized void
 804         add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 805             if (ignoreClusterDcnEventForFollower()) {
 806                 return;
 807             }
 808             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
 809             synchronized (BgpConfigurationManager.this) {
 810                 BgpRouter br = getClient(yangObj);
 811                 if (br == null) {
 812                     return;
 813                 }
 814                 String peerIp = val.getPeerIp().getValue();
 815                 try {
 816                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
 817                 } catch (Exception e) {
 818                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 819                             + addWarn);
 820                 }
 821             }
 822         }
 823
 824         @Override
 825         protected UpdateSourceReactor getDataTreeChangeListener() {
 826             return UpdateSourceReactor.this;
 827         }
 828
 829         @Override
 830         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
 831             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(UpdateSource.class);
 832         }
 833
 834         protected synchronized void
 835         remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 836             if (ignoreClusterDcnEventForFollower()) {
 837                 return;
 838             }
 839             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
 840             synchronized (BgpConfigurationManager.this) {
 841                 BgpRouter br = getClient(yangObj);
 842                 if (br == null) {
 843                     return;
 844                 }
 845                 String peerIp = val.getPeerIp().getValue();
 846                 try {
 847                     br.delUpdateSource(peerIp);
 848                 } catch (Exception e) {
 849                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 850                             + delWarn);
 851                 }
 852             }
 853         }
 854
 855         protected void update(InstanceIdentifier<UpdateSource> iid,
 856                               UpdateSource oldval, UpdateSource newval) {
 857             if (ignoreClusterDcnEventForFollower()) {
 858                 return;
 859             }
 860             LOG.error(yangObj + updWarn);
 861         }
 862
 863         @Override
 864         public void close() {
 865             try {
 866                 super.close();
 867             } catch (Exception e) {
 868                 e.printStackTrace();
 869             }
 870         }
 871     }
 872
 873     public class AddressFamiliesReactor
 874             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
 875             implements AutoCloseable, ClusteredDataTreeChangeListener<AddressFamilies> {
 876
 877         private static final String yangObj = "address-families ";
 878
 879         public AddressFamiliesReactor() {
 880             super(AddressFamilies.class, AddressFamiliesReactor.class);
 881         }
 882
 883         protected synchronized void
 884         add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
 885             if (ignoreClusterDcnEventForFollower()) {
 886                 return;
 887             }
 888             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
 889             synchronized (BgpConfigurationManager.this) {
 890                 BgpRouter br = getClient(yangObj);
 891                 if (br == null) {
 892                     return;
 893                 }
 894                 String peerIp = val.getPeerIp().getValue();
 895                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
 896                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
 897                 try {
 898                     br.addAddressFamily(peerIp, afi, safi);
 899                 } catch (Exception e) {
 900                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 901                             + addWarn);
 902                 }
 903             }
 904         }
 905
 906         @Override
 907         protected AddressFamiliesReactor getDataTreeChangeListener() {
 908             return AddressFamiliesReactor.this;
 909         }
 910
 911         @Override
 912         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
 913             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(AddressFamilies.class);
 914         }
 915
 916         protected synchronized void
 917         remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
 918             if (ignoreClusterDcnEventForFollower()) {
 919                 return;
 920             }
 921             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
 922             synchronized (BgpConfigurationManager.this) {
 923                 BgpRouter br = getClient(yangObj);
 924                 if (br == null) {
 925                     return;
 926                 }
 927                 String peerIp = val.getPeerIp().getValue();
 928                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
 929                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
 930                 try {
 931                     br.delAddressFamily(peerIp, afi, safi);
 932                 } catch (Exception e) {
 933                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 934                             + delWarn);
 935                 }
 936             }
 937         }
 938
 939         protected void update(InstanceIdentifier<AddressFamilies> iid,
 940                               AddressFamilies oldval, AddressFamilies newval) {
 941             if (ignoreClusterDcnEventForFollower()) {
 942                 return;
 943             }
 944             LOG.error(yangObj + updWarn);
 945         }
 946
 947         @Override
 948         public void close() {
 949             try {
 950                 super.close();
 951             } catch (Exception e) {
 952                 e.printStackTrace();
 953             }
 954         }
 955     }
 956
 957     public class NetworksReactor
 958             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
 959             implements AutoCloseable, ClusteredDataTreeChangeListener<Networks> {
 960
 961         private static final String yangObj = "networks ";
 962
 963         public NetworksReactor() {
 964             super(Networks.class, NetworksReactor.class);
 965         }
 966         @Override
 967         public NetworksReactor getDataTreeChangeListener() {
 968             return NetworksReactor.this;
 969         }
 970
 971         protected synchronized void
 972         add(InstanceIdentifier<Networks> iid, Networks val) {
 973             if (ignoreClusterDcnEventForFollower()) {
 974                 return;
 975             }
 976             LOG.debug("received add Networks config val {}", val.getPrefixLen());
 977             synchronized (BgpConfigurationManager.this) {
 978                 BgpRouter br = getClient(yangObj);
 979                 if (br == null) {
 980                     return;
 981                 }
 982                 String rd = val.getRd();
 983                 String pfxlen = val.getPrefixLen();
 984                 String nh = val.getNexthop().getValue();
 985                 Long label = val.getLabel();
 986                 int lbl = (label == null) ? qbgpConstants.LBL_NO_LABEL
 987                         : label.intValue();
 988                 try {
 989                     br.addPrefix(rd, pfxlen, nh, lbl);
 990                 } catch (Exception e) {
 991                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; " + addWarn);
 992                 }
 993             }
 994         }
 995
 996         @Override
 997         protected InstanceIdentifier<Networks> getWildCardPath() {
 998             return InstanceIdentifier.create(Bgp.class).child(Networks.class);
 999         }
1000
1001         protected synchronized void
1002         remove(InstanceIdentifier<Networks> iid, Networks val) {
1003             if (ignoreClusterDcnEventForFollower()) {
1004                 return;
1005             }
1006             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1007             synchronized (BgpConfigurationManager.this) {
1008                 BgpRouter br = getClient(yangObj);
1009                 if (br == null) {
1010                     return;
1011                 }
1012                 String rd = val.getRd();
1013                 String pfxlen = val.getPrefixLen();
1014                 Long label = val.getLabel();
1015                 int lbl = (label == null) ? 0 : label.intValue();
1016                 if (rd == null && lbl > 0) {
1017                     //LU prefix is being deleted.
1018                     rd = Integer.toString(lbl);
1019                 }
1020                 try {
1021                     br.delPrefix(rd, pfxlen);
1022                 } catch (Exception e) {
1023                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
1024                             + delWarn);
1025                 }
1026             }
1027         }
1028
1029         protected void update(final InstanceIdentifier<Networks> iid,
1030                               final Networks oldval, final Networks newval) {
1031             if (ignoreClusterDcnEventForFollower()) {
1032                 return;
1033             }
1034             LOG.debug("received update networks config val {}", newval.getPrefixLen());
1035             remove(iid, oldval);
1036             timer.schedule(new TimerTask() {
1037                 @Override
1038                 public void run() {
1039                     add(iid, newval);
1040                 }
1041             }, Integer.getInteger("bgp.nexthop.update.delay.in.secs", 5) * 1000);
1042         }
1043
1044         @Override
1045         public void close() {
1046             try {
1047                 super.close();
1048             } catch (Exception e) {
1049                 e.printStackTrace();
1050             }
1051         }
1052     }
1053
1054     static Timer timer = new Timer();
1055
1056     public class VrfsReactor
1057             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1058             implements AutoCloseable, ClusteredDataTreeChangeListener<Vrfs> {
1059
1060         private static final String yangObj = "vrfs ";
1061
1062         public VrfsReactor() {
1063             super(Vrfs.class, VrfsReactor.class);
1064         }
1065
1066         protected synchronized void
1067         add(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1068             if (ignoreClusterDcnEventForFollower()) {
1069                 return;
1070             }
1071             LOG.debug("received add Vrfs config val {}", val.getRd());
1072             synchronized (BgpConfigurationManager.this) {
1073                 BgpRouter br = getClient(yangObj);
1074                 if (br == null) {
1075                     return;
1076                 }
1077                 try {
1078                     br.addVrf(val.getRd(), val.getImportRts(),
1079                             val.getExportRts());
1080                 } catch (Exception e) {
1081                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
1082                             + addWarn);
1083                 }
1084             }
1085         }
1086
1087         @Override
1088         protected VrfsReactor getDataTreeChangeListener() {
1089             return VrfsReactor.this;
1090         }
1091
1092         @Override
1093         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1094             return InstanceIdentifier.create(Bgp.class).child(Vrfs.class);
1095         }
1096
1097         protected synchronized void
1098         remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1099             if (ignoreClusterDcnEventForFollower()) {
1100                 return;
1101             }
1102             LOG.debug("received remove Vrfs config val {}", val.getRd());
1103             synchronized (BgpConfigurationManager.this) {
1104                 BgpRouter br = getClient(yangObj);
1105                 if (br == null) {
1106                     return;
1107                 }
1108                 try {
1109                     br.delVrf(val.getRd());
1110                 } catch (Exception e) {
1111                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
1112                             + delWarn);
1113                 }
1114             }
1115         }
1116
1117         protected void update(InstanceIdentifier<Vrfs> iid,
1118                               Vrfs oldval, Vrfs newval) {
1119             if (ignoreClusterDcnEventForFollower()) {
1120                 return;
1121             }
1122             LOG.debug("VRFS: Update getting triggered for VRFS rd {}", oldval.getRd());
1123             LOG.error(yangObj + updWarn);
1124         }
1125
1126         @Override
1127         public void close() {
1128             try {
1129                 super.close();
1130             } catch (Exception e) {
1131                 e.printStackTrace();
1132             }
1133         }
1134     }
1135
1136     Future lastCleanupJob;
1137     Future lastReplayJobFt = null;
1138     protected void activateMIP() {
1139         try {
1140             LOG.trace("BgpReactor: Executing MIP Activate command");
1141             Process process_bgp = Runtime.getRuntime().exec("cluster ip -a sdnc_bgp_mip");
1142             Process process_os = Runtime.getRuntime().exec("cluster ip -a sdnc_os_mip");
1143             LOG.trace("bgpMIP Activated");
1144
1145         } catch (IOException io) {
1146             LOG.error("IO Exception got while activating mip:  ", io);
1147         } catch (Exception e) {
1148             LOG.error("Exception got while activating mip: ", e);
1149         }
1150     }
1151
1152     AtomicBoolean started = new AtomicBoolean(false);
1153
1154     public class BgpReactor
1155             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1156             implements AutoCloseable, ClusteredDataTreeChangeListener<Bgp> {
1157
1158         private static final String yangObj = "Bgp ";
1159
1160         public BgpReactor() {
1161             super(Bgp.class, BgpReactor.class);
1162         }
1163
1164
1165         protected synchronized void
1166         add(InstanceIdentifier<Bgp> iid, Bgp val) {
1167             LOG.error("received add Bgp config replaying the config");
1168
1169             try {
1170                 initer.await();
1171             } catch (Exception e) {
1172             }
1173             synchronized (BgpConfigurationManager.this) {
1174                 config = val;
1175                 if (ignoreClusterDcnEventForFollower()) {
1176                     return;
1177                 }
1178                 activateMIP();
1179                 if (isIpAvailable(odlThriftIp)) {
1180                     bgpRestarted();
1181                 } else {
1182                     ipActivationCheckTimer.scheduleAtFixedRate(new TimerTask() {
1183                         @Override
1184                         public void run() {
1185                             if (isIpAvailable(odlThriftIp)) {
1186                                 bgpRestarted();
1187                                 ipActivationCheckTimer.cancel();
1188                             } else {
1189                                 LOG.trace("waiting for odlThriftIP: {} to be present", odlThriftIp);
1190                             }
1191                         }
1192                     }, 10000L, 10000L);
1193                 }
1194             }
1195         }
1196
1197         @Override
1198         protected BgpReactor getDataTreeChangeListener() {
1199             return BgpReactor.this;
1200         }
1201
1202         @Override
1203         protected InstanceIdentifier<Bgp> getWildCardPath() {
1204             return InstanceIdentifier.create(Bgp.class);
1205         }
1206
1207         protected synchronized void
1208         remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1209             if (ignoreClusterDcnEventForFollower()) {
1210                 return;
1211             }
1212             LOG.debug("received remove Bgp config");
1213             synchronized (BgpConfigurationManager.this) {
1214                 config = null;
1215             }
1216         }
1217
1218         protected void update(InstanceIdentifier<Bgp> iid,
1219                               Bgp oldval, Bgp newval) {
1220             if (ignoreClusterDcnEventForFollower()) {
1221                 return;
1222             }
1223             synchronized (BgpConfigurationManager.this) {
1224                 config = newval;
1225             }
1226         }
1227
1228         @Override
1229         public void close() {
1230             try {
1231                 super.close();
1232             } catch (Exception e) {
1233                 e.printStackTrace();
1234             }
1235         }
1236     }
1237
1238     public void readOdlThriftIpForBgpCommunication() {
1239         File f = new File(CLUSTER_CONF_FILE);
1240         if (!f.exists()) {
1241             odlThriftIp = "127.0.0.1";
1242             return;
1243         }
1244         BufferedReader br = null;
1245         try {
1246             br = new BufferedReader(new InputStreamReader(
1247                     new FileInputStream(f)));
1248             String line = br.readLine();
1249             while (line != null) {
1250                 if (line.contains(SDNC_BGP_MIP)) {
1251                     line = line.trim();
1252                     odlThriftIp = line.substring(line.lastIndexOf(" ") + 1);
1253                     break;
1254                 }
1255                 line = br.readLine();
1256             }
1257         } catch (Exception e) {
1258         } finally {
1259             try {
1260                 br.close();
1261             } catch (Exception ignore) {
1262             }
1263         }
1264     }
1265
1266     public boolean isIpAvailable(String odlip) {
1267
1268         try {
1269             if (odlip != null) {
1270                 if ("127.0.0.1".equals(odlip)) {
1271                     return true;
1272                 }
1273                 Enumeration e = NetworkInterface.getNetworkInterfaces();
1274                 while (e.hasMoreElements()) {
1275                     NetworkInterface n = (NetworkInterface) e.nextElement();
1276                     Enumeration ee = n.getInetAddresses();
1277                     while (ee.hasMoreElements()) {
1278                         InetAddress i = (InetAddress) ee.nextElement();
1279                         if (odlip.equals(i.getHostAddress())) {
1280                             return true;
1281                         }
1282                         ;
1283                     }
1284                 }
1285             }
1286         } catch (Exception e) {
1287         }
1288         return false;
1289     }
1290
1291     public static long getStalePathtime(int defValue, AsId as_num) {
1292         long spt = 0;
1293         try {
1294             spt = bgpManager.getConfig().getGracefulRestart().getStalepathTime();
1295         } catch (Exception e) {
1296             try {
1297                 spt = as_num.getStalepathTime();
1298                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1299             } catch (Exception ignore) {
1300                 LOG.trace("BGP AS id is not set using graceful");
1301                 spt = defValue;
1302             }
1303         }
1304         if (spt == 0) {
1305             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1306             spt = defValue;
1307         }
1308         return spt;
1309     }
1310
1311     public synchronized void bgpRestarted() {
1312         /*
1313          * If there a thread which in the process of stale cleanup, cancel it
1314          * and start a new thread (to avoid processing same again).
1315          */
1316         if (previousReplayJobInProgress()) {
1317             cancelPreviousReplayJob();
1318         }
1319         Runnable task = new Runnable() {
1320             @Override
1321             public void run() {
1322                 try {
1323                     LOG.error("running bgp replay task ");
1324                     if (get() == null) {
1325                         String host = getConfigHost();
1326                         int port = getConfigPort();
1327                         LOG.info("connecting  to bgp host {} ", host);
1328
1329                         boolean res = bgpRouter.connect(host, port);
1330                         LOG.error("no config to push in bgp replay task ");
1331                         return;
1332                     }
1333                     setStaleStartTime(System.currentTimeMillis());
1334                     LOG.error("started creating stale fib  map ");
1335                     createStaleFibMap();
1336                     setStaleEndTime(System.currentTimeMillis());
1337                     LOG.error("took {} msecs for stale fib map creation ", getStaleEndTime()- getStaleStartTime());
1338                     LOG.error("started bgp config replay ");
1339                     setCfgReplayStartTime(System.currentTimeMillis());
1340                     replay();
1341                     setCfgReplayEndTime(System.currentTimeMillis());
1342                     LOG.error("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1343                     long route_sync_time = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1344                     Thread.sleep(route_sync_time * 1000L);
1345                     setStaleCleanupTime(route_sync_time);
1346                     new RouteCleanup().call();
1347                 } catch (Exception eCancel) {
1348                     LOG.error("Stale Cleanup Task Cancelled", eCancel);
1349                 }
1350             }
1351         };
1352         lastReplayJobFt = executor.submit(task);
1353     }
1354
1355     private boolean previousReplayJobInProgress() {
1356         return lastReplayJobFt != null && !lastReplayJobFt.isDone();
1357     }
1358
1359     private void cancelPreviousReplayJob() {
1360         try {
1361             LOG.error("cancelling already running bgp replay task");
1362             lastReplayJobFt.cancel(true);
1363             lastReplayJobFt = null;
1364             Thread.sleep(2000);
1365         } catch (Throwable e) {
1366             LOG.error("Failed to cancel previous replay job ",e);
1367         }
1368     }
1369
1370     private static void doRouteSync() {
1371         BgpSyncHandle bsh = BgpSyncHandle.getInstance();
1372         LOG.error("Starting BGP route sync");
1373         try {
1374             bgpRouter.initRibSync(bsh);
1375         } catch (Exception e) {
1376             LOG.error("Route sync aborted, exception when initializing: " + e);
1377             return;
1378         }
1379         while (bsh.getState() != bsh.DONE) {
1380             Routes routes = null;
1381             try {
1382                 routes = bgpRouter.doRibSync(bsh);
1383             } catch (Exception e) {
1384                 LOG.error("Route sync aborted, exception when syncing: " + e);
1385                 return;
1386             }
1387             Iterator<Update> updates = routes.getUpdatesIterator();
1388             while (updates.hasNext()) {
1389                 Update u = updates.next();
1390                 Map<String, Map<String, String>> stale_fib_rd_map = BgpConfigurationManager.getStaledFibEntriesMap();
1391                 String rd = u.getRd();
1392                 String nexthop = u.getNexthop();
1393                 int label = u.getLabel();
1394                 String prefix = u.getPrefix();
1395                 int plen = u.getPrefixlen();
1396                 onUpdatePushRoute(rd, prefix, plen, nexthop, label);
1397             }
1398         }
1399         try {
1400             LOG.error("Ending BGP route-sync");
1401             bgpRouter.endRibSync(bsh);
1402         } catch (Exception e) {
1403         }
1404     }
1405
1406     /* onUpdatePushRoute
1407      * Get Stale fib map, and compare current route/fib entry.
1408      *  - Entry compare shall include NextHop, Label.
1409      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1410      *  - If entry not found, add to FIB Config DS.
1411      *  - If entry found, but either Label/NextHop doesn't match.
1412      *      - Update FIB Config DS with modified values.
1413      *      - delete from Stale Map.
1414      */
1415     public static void onUpdatePushRoute(String rd, String prefix, int plen, String nextHop, int label) {
1416         Map<String, Map<String, String>> stale_fib_rd_map = BgpConfigurationManager.getStaledFibEntriesMap();
1417         boolean addroute = false;
1418         if (!stale_fib_rd_map.isEmpty()) {
1419             // restart Scenario, as MAP is not empty.
1420             Map<String, String> map = stale_fib_rd_map.get(rd);
1421             if (map != null) {
1422                 String nexthoplabel = map.get(prefix + "/" + plen);
1423                 if (null == nexthoplabel) {
1424                     // New Entry, which happened to be added during restart.
1425                     addroute = true;
1426                 } else {
1427                     map.remove(prefix + "/" + plen);
1428                     if (isRouteModified(nextHop, label, nexthoplabel)) {
1429                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1430                         // Existing entry, where in Nexthop/Label got modified during restart
1431                         addroute = true;
1432                     }
1433                 }
1434             }
1435         } else {
1436             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1437             addroute = true;
1438         }
1439         if (addroute) {
1440             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1441             fib.addFibEntryToDS(rd, prefix + "/" + plen, Arrays.asList(nextHop), label, RouteOrigin.BGP);
1442             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1443         }
1444     }
1445
1446     private static boolean isRouteModified(String nexthop, int label, String nexthoplabel) {
1447         return !nexthoplabel.isEmpty() && !nexthoplabel.equals(nexthop + "/" + label);
1448     }
1449
1450     static private void replayNbrConfig(List<Neighbors> n, BgpRouter br) {
1451         for (Neighbors nbr : n) {
1452             try {
1453                 br.addNeighbor(nbr.getAddress().getValue(),
1454                         nbr.getRemoteAs().intValue());
1455                 //itmProvider.buildTunnelsToDCGW(new IpAddress(nbr.getAddress().getValue().toCharArray()));
1456             } catch (Exception e) {
1457                 LOG.error("Replay:addNbr() received exception: \"" + e + "\"");
1458                 continue;
1459             }
1460             EbgpMultihop en = nbr.getEbgpMultihop();
1461             if (en != null) {
1462                 try {
1463                     br.addEbgpMultihop(en.getPeerIp().getValue(),
1464                             en.getNhops().intValue());
1465                 } catch (Exception e) {
1466                     LOG.error("Replay:addEBgp() received exception: \"" + e + "\"");
1467                 }
1468             }
1469             UpdateSource us = nbr.getUpdateSource();
1470             if (us != null) {
1471                 try {
1472                     br.addUpdateSource(us.getPeerIp().getValue(),
1473                             us.getSourceIp().getValue());
1474                 } catch (Exception e) {
1475                     LOG.error("Replay:addUS() received exception: \"" + e + "\"");
1476                 }
1477             }
1478             List<AddressFamilies> afs = nbr.getAddressFamilies();
1479             if (afs != null) {
1480                 for (AddressFamilies af : afs) {
1481                     af_afi afi = af_afi.findByValue(af.getAfi().intValue());
1482                     af_safi safi = af_safi.findByValue(af.getSafi().intValue());
1483                     try {
1484                         br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
1485                     } catch (Exception e) {
1486                         LOG.error("Replay:addAf() received exception: \"" + e + "\"");
1487                     }
1488                 }
1489             }
1490         }
1491     }
1492
1493     public static String getConfigHost() {
1494         if (config == null) {
1495             return cHostStartup;
1496         }
1497         ConfigServer ts = config.getConfigServer();
1498         return (ts == null ? cHostStartup : ts.getHost().getValue());
1499     }
1500
1501     public static int getConfigPort() {
1502         if (config == null) {
1503             return Integer.parseInt(cPortStartup);
1504         }
1505         ConfigServer ts = config.getConfigServer();
1506         return (ts == null ? Integer.parseInt(cPortStartup) :
1507                 ts.getPort().intValue());
1508     }
1509
1510     public static synchronized void replay() {
1511         synchronized (bgpConfigurationManager) {
1512             String host = getConfigHost();
1513             int port = getConfigPort();
1514             LOG.error("connecting  to bgp host {} ", host);
1515
1516             boolean res = bgpRouter.connect(host, port);
1517             if (!res) {
1518                 String msg = "Cannot connect to BGP config server at " + host + ":" + port;
1519                 if (config != null) {
1520                     msg += "; Configuration Replay aborted";
1521                 }
1522                 LOG.error(msg);
1523                 return;
1524             }
1525             config = bgpManager.getConfig();
1526             if (config == null) {
1527                 LOG.error("bgp config is empty nothing to push to bgp");
1528                 return;
1529             }
1530             BgpRouter br = bgpRouter;
1531             AsId a = config.getAsId();
1532             if (a == null) {
1533                 return;
1534             }
1535             int asNum = a.getLocalAs().intValue();
1536             Ipv4Address routerId = a.getRouterId();
1537             Long spt = a.getStalepathTime();
1538             Boolean afb = a.isAnnounceFbit();
1539             String rid = (routerId == null) ? "" : routerId.getValue();
1540             int stalepathTime = (int) getStalePathtime(0, config.getAsId());
1541             boolean announceFbit = (afb == null) ? false : afb.booleanValue();
1542             try {
1543                 br.startBgp(asNum, rid, stalepathTime, announceFbit);
1544             } catch (BgpRouterException bre) {
1545                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
1546                     doRouteSync();
1547                 } else {
1548                     LOG.error("Replay: startBgp() received exception: \""
1549                             + bre + "\"; " + addWarn);
1550                 }
1551             } catch (Exception e) {
1552                 //not unusual. We may have restarted & BGP is already on
1553                 LOG.error("Replay:startBgp() received exception: \"" + e + "\"");
1554             }
1555
1556             if (bgpManager.getBgpCounters() == null) {
1557                 bgpManager.startBgpCountersTask();
1558             }
1559
1560             Logging l = config.getLogging();
1561             if (l != null) {
1562                 try {
1563                     br.setLogging(l.getFile(), l.getLevel());
1564                 } catch (Exception e) {
1565                     LOG.error("Replay:setLogging() received exception: \"" + e + "\"");
1566                 }
1567             }
1568
1569             GracefulRestart g = config.getGracefulRestart();
1570             if (g != null) {
1571                 try {
1572                     br.addGracefulRestart(g.getStalepathTime().intValue());
1573                 } catch (Exception e) {
1574                     LOG.error("Replay:addGr() received exception: \"" + e + "\"");
1575                 }
1576             }
1577
1578             List<Neighbors> n = config.getNeighbors();
1579             if (n != null) {
1580                 LOG.error("configuring existing Neighbors present for replay total neighbors {}", n.size());
1581                 replayNbrConfig(n, br);
1582             } else {
1583                 LOG.error("no Neighbors present for replay config ");
1584             }
1585
1586             List<Vrfs> v = config.getVrfs();
1587             if (v != null) {
1588                 for (Vrfs vrf : v) {
1589                     try {
1590                         br.addVrf(vrf.getRd(), vrf.getImportRts(),
1591                                 vrf.getExportRts());
1592                     } catch (Exception e) {
1593                         LOG.error("Replay:addVrf() received exception: \"" + e + "\"");
1594                     }
1595                 }
1596             }
1597
1598             List<Networks> ln = config.getNetworks();
1599             if (ln != null) {
1600                 for (Networks net : ln) {
1601                     String rd = net.getRd();
1602                     String pfxlen = net.getPrefixLen();
1603                     String nh = net.getNexthop().getValue();
1604                     Long label = net.getLabel();
1605                     int lbl = (label == null) ? 0 : label.intValue();
1606                     if (rd == null && lbl > 0) {
1607                         //LU prefix is being deleted.
1608                         rd = Integer.toString(lbl);
1609                     }
1610                     try {
1611                         br.addPrefix(rd, pfxlen, nh, lbl);
1612                     } catch (Exception e) {
1613                         LOG.error("Replay:addPfx() received exception: \"" + e + "\"");
1614                     }
1615                 }
1616             }
1617         }
1618     }
1619
1620     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
1621         BgpUtil.update(broker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1622     }
1623
1624     private <T extends DataObject> void asyncWrite(InstanceIdentifier<T> iid, T dto) {
1625         BgpUtil.write(broker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1626     }
1627
1628     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
1629         BgpUtil.delete(broker, LogicalDatastoreType.CONFIGURATION, iid);
1630     }
1631
1632     public synchronized void
1633     startConfig(String bgpHost, int thriftPort) {
1634         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
1635                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
1636         InstanceIdentifier<ConfigServer> iid = iib.build();
1637         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
1638         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
1639                 .setPort((long) thriftPort).build();
1640         update(iid, dto);
1641     }
1642
1643     public synchronized void
1644     startBgp(int as, String routerId, int spt, boolean fbit) {
1645         Long localAs = (long) as;
1646         Ipv4Address rid = (routerId == null) ?
1647                 null : new Ipv4Address(routerId);
1648         Long staleTime = (long) spt;
1649         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
1650                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
1651         InstanceIdentifier<AsId> iid = iib.build();
1652         AsId dto = new AsIdBuilder().setLocalAs(localAs)
1653                 .setRouterId(rid)
1654                 .setStalepathTime(staleTime)
1655                 .setAnnounceFbit(fbit).build();
1656         update(iid, dto);
1657     }
1658
1659     public synchronized void
1660     addLogging(String fileName, String logLevel) {
1661         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
1662                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
1663         InstanceIdentifier<Logging> iid = iib.build();
1664         Logging dto = new LoggingBuilder().setFile(fileName)
1665                 .setLevel(logLevel).build();
1666         update(iid, dto);
1667     }
1668
1669     public synchronized void
1670     addGracefulRestart(int staleTime) {
1671         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
1672                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
1673         InstanceIdentifier<GracefulRestart> iid = iib.build();
1674         GracefulRestart dto = new GracefulRestartBuilder()
1675                 .setStalepathTime((long) staleTime).build();
1676         update(iid, dto);
1677     }
1678
1679     public synchronized void
1680     addNeighbor(String nbrIp, int remoteAs) {
1681         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1682         Long rAs = (long) remoteAs;
1683         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
1684                 InstanceIdentifier.builder(Bgp.class)
1685                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
1686         InstanceIdentifier<Neighbors> iid = iib.build();
1687         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
1688                 .setRemoteAs(rAs).build();
1689         update(iid, dto);
1690     }
1691
1692     public synchronized void
1693     addUpdateSource(String nbrIp, String srcIp) {
1694         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1695         Ipv4Address srcAddr = new Ipv4Address(srcIp);
1696         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
1697                 InstanceIdentifier.builder(Bgp.class)
1698                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1699                         .child(UpdateSource.class);
1700         InstanceIdentifier<UpdateSource> iid = iib.build();
1701         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
1702                 .setSourceIp(srcAddr).build();
1703         update(iid, dto);
1704     }
1705
1706     public synchronized void
1707     addEbgpMultihop(String nbrIp, int nHops) {
1708         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1709         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
1710                 InstanceIdentifier.builder(Bgp.class)
1711                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1712                         .child(EbgpMultihop.class);
1713         InstanceIdentifier<EbgpMultihop> iid = iib.build();
1714         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
1715                 .setNhops((long) nHops).build();
1716         update(iid, dto);
1717     }
1718
1719     public synchronized void
1720     addAddressFamily(String nbrIp, int afi, int safi) {
1721         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1722         Long a = (long) afi;
1723         Long sa = (long) safi;
1724         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
1725                 InstanceIdentifier.builder(Bgp.class)
1726                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1727                         .child(AddressFamilies.class, new AddressFamiliesKey(a, sa));
1728         InstanceIdentifier<AddressFamilies> iid = iib.build();
1729         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
1730                 .setAfi(a).setSafi(sa).build();
1731         update(iid, dto);
1732     }
1733
1734     public synchronized void
1735     addPrefix(String rd, String pfx, List<String> nhList, int lbl) {
1736         for (String nh : nhList) {
1737             Ipv4Address nexthop = new Ipv4Address(nh);
1738             Long label = (long) lbl;
1739             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
1740                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
1741             Networks dto = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
1742                                                 .setLabel(label).build();
1743             update(iid, dto);
1744         }
1745     }
1746
1747     public synchronized void
1748     addVrf(String rd, List<String> irts, List<String> erts) {
1749         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
1750                 InstanceIdentifier.builder(Bgp.class)
1751                         .child(Vrfs.class, new VrfsKey(rd));
1752         InstanceIdentifier<Vrfs> iid = iib.build();
1753         Vrfs dto = new VrfsBuilder().setRd(rd)
1754                 .setImportRts(irts)
1755                 .setExportRts(erts).build();
1756
1757         asyncWrite(iid, dto);
1758     }
1759
1760     public synchronized void stopConfig() {
1761         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
1762                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
1763         InstanceIdentifier<ConfigServer> iid = iib.build();
1764         delete(iid);
1765     }
1766
1767     public synchronized void stopBgp() {
1768         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
1769                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
1770         InstanceIdentifier<AsId> iid = iib.build();
1771         delete(iid);
1772     }
1773
1774     public synchronized void delLogging() {
1775         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
1776                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
1777         InstanceIdentifier<Logging> iid = iib.build();
1778         delete(iid);
1779     }
1780
1781     public synchronized void delGracefulRestart() {
1782         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
1783                 InstanceIdentifier.builder(Bgp.class)
1784                         .child(GracefulRestart.class);
1785         InstanceIdentifier<GracefulRestart> iid = iib.build();
1786         delete(iid);
1787     }
1788
1789     public synchronized void delNeighbor(String nbrIp) {
1790         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1791         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
1792                 InstanceIdentifier.builder(Bgp.class)
1793                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
1794         InstanceIdentifier<Neighbors> iid = iib.build();
1795         delete(iid);
1796     }
1797
1798     public synchronized void delUpdateSource(String nbrIp) {
1799         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1800         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
1801                 InstanceIdentifier.builder(Bgp.class)
1802                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1803                         .child(UpdateSource.class);
1804         InstanceIdentifier<UpdateSource> iid = iib.build();
1805         delete(iid);
1806     }
1807
1808     public synchronized void delEbgpMultihop(String nbrIp) {
1809         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1810         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
1811                 InstanceIdentifier.builder(Bgp.class)
1812                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1813                         .child(EbgpMultihop.class);
1814         InstanceIdentifier<EbgpMultihop> iid = iib.build();
1815         delete(iid);
1816     }
1817
1818     public synchronized void
1819     delAddressFamily(String nbrIp, int afi, int safi) {
1820         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1821         Long a = (long) afi;
1822         Long sa = (long) safi;
1823         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
1824                 InstanceIdentifier.builder(Bgp.class)
1825                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1826                         .child(AddressFamilies.class, new AddressFamiliesKey(a, sa));
1827         InstanceIdentifier<AddressFamilies> iid = iib.build();
1828         delete(iid);
1829     }
1830
1831     public synchronized void delPrefix(String rd, String pfx) {
1832         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
1833                 InstanceIdentifier.builder(Bgp.class)
1834                         .child(Networks.class, new NetworksKey(pfx, rd));
1835         InstanceIdentifier<Networks> iid = iib.build();
1836         delete(iid);
1837     }
1838
1839     public synchronized void delVrf(String rd) {
1840         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
1841                 InstanceIdentifier.builder(Bgp.class)
1842                         .child(Vrfs.class, new VrfsKey(rd));
1843         InstanceIdentifier<Vrfs> iid = iib.build();
1844         delete(iid);
1845     }
1846
1847     static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
1848
1849     /*
1850     * Remove Stale Marked Routes after timer expiry.
1851     */
1852     class RouteCleanup implements Callable<Integer> {
1853
1854         @Override
1855         public Integer call() {
1856             totalCleared = 0;
1857             try {
1858                 if (staledFibEntriesMap.isEmpty()) {
1859                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
1860                 } else {
1861                     for (String rd : staledFibEntriesMap.keySet()) {
1862                         if (Thread.interrupted()) {
1863                             return 0;
1864                         }
1865                         Map<String, String> map = staledFibEntriesMap.get(rd);
1866                         if (map != null) {
1867                             for (String prefix : map.keySet()) {
1868                                 if (Thread.interrupted()) {
1869                                     return 0;
1870                                 }
1871                                 try {
1872                                     totalCleared++;
1873                                     LOG.error("BGP: RouteCleanup deletePrefix called but not executed rd:{}, prefix{}" + rd.toString() + prefix);
1874                                     // fib.removeFibEntryFromDS(rd, prefix);
1875                                 } catch (Exception e) {
1876                                     LOG.error("BGP: RouteCleanup deletePrefix failed rd:{}, prefix{}" + rd.toString() + prefix);
1877                                 }
1878                             }
1879                         }
1880                     }
1881                 }
1882             } catch (Exception e) {
1883                 LOG.error("Cleanup Thread Got interrupted, Failed to cleanup stale routes ", e);
1884             } finally {
1885                 staledFibEntriesMap.clear();
1886             }
1887             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
1888             return 0;
1889         }
1890     }
1891
1892     /*
1893      * BGP restart scenario, ODL-BGP manager was/is running.
1894      * On re-sync notification, Get a copy of FIB database.
1895      */
1896     public static void createStaleFibMap() {
1897         totalStaledCount = 0;
1898         try {
1899             /*
1900             * at the time Stale FIB creation, Wait till all PENDING write transaction
1901              * to complete (or)wait for max timeout value of STALE_FIB_WAIT Seconds.
1902              */
1903             int retry = STALE_FIB_WAIT;
1904             while ((BgpUtil.getGetPendingWrTransaction() != 0) && (retry > 0)) {
1905                 Thread.sleep(1000);
1906                 retry--;
1907                 if (retry == 0) {
1908                     LOG.error("TimeOut occured {} seconds, in waiting stale fib create", STALE_FIB_WAIT);
1909                 }
1910             }
1911             staledFibEntriesMap.clear();
1912             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
1913             DataBroker db = BgpUtil.getBroker();
1914             if (db == null) {
1915                 LOG.error("Couldn't find BgpUtil broker while creating createStaleFibMap");
1916                 return;
1917             }
1918
1919             Optional<FibEntries> fibEntries = BgpUtil.read(BgpUtil.getBroker(),
1920                     LogicalDatastoreType.CONFIGURATION, id);
1921             if (fibEntries.isPresent()) {
1922                 List<VrfTables> stale_vrfTables = fibEntries.get().getVrfTables();
1923                 for (VrfTables vrfTable : stale_vrfTables) {
1924                     Map<String, String> stale_fib_ent_map = new HashMap<>();
1925                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
1926                         if (Thread.interrupted()) {
1927                             break;
1928                         }
1929                         totalStaledCount++;
1930                         //Create MAP from stale_vrfTables.
1931                         //FIXME: Once odl-fib.yang is updated with latest yang to contain nexthopaddresslist
1932 //                        for (String nextHop : vrfEntry.getNextHopAddressList()) {
1933 //                            stale_fib_ent_map.put(vrfEntry.getDestPrefix(), nextHop + "/" + vrfEntry.getLabel());
1934 //                        }
1935
1936                     }
1937                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), stale_fib_ent_map);
1938                 }
1939             } else {
1940                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
1941             }
1942         } catch (Exception e) {
1943             LOG.error("createStaleFibMap:: error ", e);
1944         }
1945         LOG.error("created {} staled entries ", totalStaledCount);
1946     }
1947
1948     //map<rd, map<prefix/len, nexthop/label>>
1949     public static Map<String, Map<String, String>> getStaledFibEntriesMap() {
1950         return staledFibEntriesMap;
1951     }
1952
1953     //TODO: below function is for testing purpose with cli
1954     public static void onUpdateWithdrawRoute(String rd, String prefix, int plen) {
1955         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
1956         try {
1957             fib.removeFibEntryFromDS(rd, prefix + "/" + plen);
1958         } catch (Throwable e) {
1959             LOG.error("failed to handle withdraw route ", e);
1960         }
1961     }
1962     public boolean isBgpConnected(){
1963         return bgpRouter.isBgpConnected();
1964     }
1965     public long getLastConnectedTS() {
1966         return bgpRouter.getLastConnectedTS();
1967     }
1968     public long getConnectTS() {
1969         return bgpRouter.getConnectTS();
1970     }
1971     public long getStartTS() {
1972         return bgpRouter.getStartTS();
1973     }
1974     public static int getTotalStaledCount() {return totalStaledCount;}
1975     public static int getTotalCleared() { return totalCleared;}
1976 }