vpnservice/bgpmanager/bgpmanager-impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright (c) 2015 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import com.google.common.base.Optional;
  11 import io.netty.util.concurrent.GlobalEventExecutor;
  12 import java.io.BufferedReader;
  13 import java.io.File;
  14 import java.io.FileInputStream;
  15 import java.io.IOException;
  16 import java.io.InputStreamReader;
  17 import java.lang.reflect.Constructor;
  18 import java.net.InetAddress;
  19 import java.net.NetworkInterface;
  20 import java.util.Arrays;
  21 import java.util.Enumeration;
  22 import java.util.HashMap;
  23 import java.util.Iterator;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Timer;
  27 import java.util.TimerTask;
  28 import java.util.concurrent.Callable;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.CountDownLatch;
  31 import java.util.concurrent.ExecutionException;
  32 import java.util.concurrent.Executors;
  33 import java.util.concurrent.Future;
  34 import java.util.concurrent.ScheduledExecutorService;
  35 import java.util.concurrent.TimeoutException;
  36 import java.util.concurrent.atomic.AtomicBoolean;
  37 import org.opendaylight.controller.config.api.osgi.WaitingServiceTracker;
  38 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  41 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipChange;
  42 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipListener;
  43 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipService;
  44 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  45 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  46 import org.opendaylight.genius.utils.clustering.EntityOwnerUtils;
  47 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  48 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  49 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  50 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  51 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  52 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  53 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  54 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  55 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  56 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  57 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  58 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  59 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  60 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  61 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  62 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  63 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  64 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
  65 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
  67 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
  69 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
  71 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Neighbors;
  73 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsKey;
  75 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Networks;
  76 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksKey;
  78 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Vrfs;
  79 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsBuilder;
  80 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsKey;
  81 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamilies;
  82 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesKey;
  84 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihop;
  85 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihopBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSource;
  87 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSourceBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  92 import org.opendaylight.yangtools.concepts.ListenerRegistration;
  93 import org.opendaylight.yangtools.yang.binding.DataObject;
  94 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  95 import org.opendaylight.genius.utils.batching.DefaultBatchHandler;
  96 import org.osgi.framework.BundleContext;
  97 import org.slf4j.Logger;
  98 import org.slf4j.LoggerFactory;
  99 import java.util.concurrent.atomic.AtomicInteger;
 100 import org.opendaylight.yangtools.yang.binding.DataObject;
 101
 102 public class BgpConfigurationManager {
 103     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 104     private static DataBroker dataBroker;
 105     private static FibDSWriter fibDSWriter;
 106     public static IBgpManager bgpManager;
 107     private final BundleContext bundleContext;
 108     private static Bgp config;
 109     private static BgpRouter bgpRouter;
 110     private static BgpThriftService updateServer;
 111     private BgpCounters bgpCounters;
 112     private BgpAlarms bgpAlarms;
 113     private Timer bgpCountersTimer;
 114     private Timer bgpAlarmsTimer;
 115     private static final String DEF_LOGFILE = "/var/log/bgp_debug.log";
 116     private static final String DEF_LOGLEVEL = "errors";
 117     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 118     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 119     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 120     private static final String DEF_UPORT = "6644";
 121     private static final String DEF_CHOST = "127.0.0.1";
 122     private static final String DEF_CPORT = "7644";
 123     private static final String SDNC_BGP_MIP = "sdnc_bgp_mip";
 124     private static final String BGP_SDNC_MIP = "bgp_sdnc_mip";
 125     private static final String CLUSTER_CONF_FILE = "/cluster/etc/cluster.conf";
 126     private static final Timer ipActivationCheckTimer = new Timer();
 127     private static final int STALE_FIB_WAIT = 60;
 128     private static final int RESTART_DEFAULT_GR = 90;
 129     private long StaleStartTime = 0;
 130     private long StaleEndTime = 0;
 131     private long CfgReplayStartTime = 0;
 132     private long CfgReplayEndTime = 0;
 133     private long StaleCleanupTime = 0;
 134     private static final int dsRetryCoount = 100; //100 retries, each after waitTimeBetweenEachTryMillis seconds
 135     private static final long waitTimeBetweenEachTryMillis = 1000L; //one second sleep after every retry
 136
 137     public String getBgpSdncMipIp() { return readThriftIpForCommunication(BGP_SDNC_MIP);}
 138     public long getStaleCleanupTime() {
 139         return StaleCleanupTime;
 140     }
 141
 142     public void setStaleCleanupTime(long staleCleanupTime) {
 143         StaleCleanupTime = staleCleanupTime;
 144     }
 145
 146     public long getCfgReplayEndTime() {
 147         return CfgReplayEndTime;
 148     }
 149
 150     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 151         CfgReplayEndTime = cfgReplayEndTime;
 152     }
 153
 154     public long getCfgReplayStartTime() {
 155         return CfgReplayStartTime;
 156     }
 157
 158     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 159         CfgReplayStartTime = cfgReplayStartTime;
 160     }
 161
 162     public long getStaleEndTime() {
 163         return StaleEndTime;
 164     }
 165
 166     public void setStaleEndTime(long staleEndTime) {
 167         StaleEndTime = staleEndTime;
 168     }
 169
 170     public long getStaleStartTime() {
 171         return StaleStartTime;
 172     }
 173
 174     public void setStaleStartTime(long staleStartTime) {
 175         StaleStartTime = staleStartTime;
 176     }
 177
 178
 179     // to have stale FIB map (RD, Prefix)
 180     //  number of seconds wait for route sync-up between ODL and BGP.
 181     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 182
 183     static String odlThriftIp = "127.0.0.1";
 184     static String bgpThriftIp = "127.0.0.1";
 185     private static String cHostStartup;
 186     private static String cPortStartup;
 187     private static CountDownLatch initer = new CountDownLatch(1);
 188     //static IITMProvider itmProvider;
 189     //map<rd, map<prefix/len, nexthop/label>>
 190     private static Map<String, Map<String, String>> staledFibEntriesMap = new ConcurrentHashMap<>();
 191
 192     static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 193     static final String BGP_ENTITY_NAME = "bgp";
 194
 195     static int totalStaledCount = 0;
 196     static int totalCleared = 0;
 197
 198     private static final Class[] reactors = {
 199             ConfigServerReactor.class, AsIdReactor.class,
 200             GracefulRestartReactor.class, LoggingReactor.class,
 201             NeighborsReactor.class, UpdateSourceReactor.class,
 202             EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 203             NetworksReactor.class, VrfsReactor.class, BgpReactor.class
 204     };
 205
 206     private ListenerRegistration<DataChangeListener>[] registrations;
 207
 208     final BgpConfigurationManager bgpConfigurationManager;
 209
 210     public BgpConfigurationManager(final DataBroker dataBroker,
 211                                    final EntityOwnershipService entityOwnershipService,
 212                                    final FibDSWriter fibDSWriter,
 213                                    final BundleContext bundleContext)
 214             throws InterruptedException, ExecutionException, TimeoutException {
 215         BgpConfigurationManager.dataBroker = dataBroker;
 216         BgpConfigurationManager.fibDSWriter = fibDSWriter;
 217         this.bundleContext = bundleContext;
 218         String uPort = getProperty(UPDATE_PORT, DEF_UPORT);
 219         cHostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 220         cPortStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 221         LOG.info("UpdateServer at localhost:" + uPort + " ConfigServer at "
 222                 + cHostStartup + ":" + cPortStartup);
 223         VtyshCli.setHostAddr(cHostStartup);
 224         ClearBgpCli.setHostAddr(cHostStartup);
 225         setEntityOwnershipService(entityOwnershipService);
 226         bgpRouter = BgpRouter.getInstance();
 227         odlThriftIp = readThriftIpForCommunication(SDNC_BGP_MIP);
 228         bgpThriftIp = readThriftIpForCommunication(BGP_SDNC_MIP);
 229         registerCallbacks();
 230
 231         LOG.info("BGP Configuration manager initialized");
 232         initer.countDown();
 233
 234         bgpConfigurationManager = this;
 235         BgpUtil.batchSize = BgpUtil.BATCH_SIZE;
 236         if (Integer.getInteger("batch.size") != null) {
 237             BgpUtil.batchSize = Integer.getInteger("batch.size");
 238         }
 239         BgpUtil.batchInterval = BgpUtil.PERIODICITY;
 240         if (Integer.getInteger("batch.wait.time") != null) {
 241             BgpUtil.batchInterval = Integer.getInteger("batch.wait.time");
 242         }
 243         BgpUtil.registerWithBatchManager(new DefaultBatchHandler(dataBroker, LogicalDatastoreType.CONFIGURATION, BgpUtil.batchSize, BgpUtil.batchInterval));
 244
 245         GlobalEventExecutor.INSTANCE.execute(new Runnable() {
 246             @Override
 247             public void run() {
 248                 final WaitingServiceTracker<IBgpManager> tracker = WaitingServiceTracker.create(
 249                         IBgpManager.class, bundleContext);
 250                 bgpManager = tracker.waitForService(WaitingServiceTracker.FIVE_MINUTES);
 251                 updateServer = new BgpThriftService(Integer.parseInt(uPort), bgpManager, fibDSWriter);
 252                 updateServer.start();
 253                 LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 254             }
 255         });
 256     }
 257
 258     private Object createListener(Class<?> cls) {
 259         Constructor<?> ctor;
 260         Object obj = null;
 261
 262         try {
 263             ctor = cls.getConstructor(BgpConfigurationManager.class);
 264             obj = ctor.newInstance(this);
 265         } catch (Exception e) {
 266             LOG.error("Failed to create listener object", e);
 267         }
 268         return obj;
 269     }
 270
 271     private void registerCallbacks() {
 272         String emsg = "Failed to register listener";
 273         registrations = new ListenerRegistration[reactors.length];
 274         InstanceIdentifier<?> iid = InstanceIdentifier.create(Bgp.class);
 275         for (Class reactor : reactors) {
 276             Object obj = createListener(reactor);
 277             String dclName = obj.getClass().getName();
 278             try {
 279                 AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 280                 dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 281             } catch (Exception e) {
 282                 LOG.error(emsg, e);
 283                 throw new IllegalStateException(emsg + " " + dclName, e);
 284             }
 285         }
 286     }
 287
 288     public void close() {
 289         if (updateServer != null) {
 290             updateServer.stop();
 291         }
 292         LOG.info("{} close", getClass().getSimpleName());
 293     }
 294
 295     private boolean configExists() throws InterruptedException, ExecutionException, TimeoutException {
 296         InstanceIdentifier.InstanceIdentifierBuilder<Bgp> iib =
 297                 InstanceIdentifier.builder(Bgp.class);
 298         InstanceIdentifier<Bgp> iid = iib.build();
 299         Optional<Bgp> b = BgpUtil.read(dataBroker,
 300                 LogicalDatastoreType.CONFIGURATION, iid);
 301         return b.isPresent();
 302     }
 303
 304     private String getProperty(String var, String def) {
 305         String s = bundleContext.getProperty(var);
 306         return (s == null ? def : s);
 307     }
 308
 309     boolean ignoreClusterDcnEventForFollower() {
 310         return !EntityOwnerUtils.amIEntityOwner(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME);
 311     }
 312
 313     public Bgp get() {
 314         config = getConfig();
 315         return config;
 316     }
 317
 318     public void setEntityOwnershipService(final EntityOwnershipService entityOwnershipService) {
 319         try {
 320             EntityOwnerUtils.registerEntityCandidateForOwnerShip(entityOwnershipService,
 321                     BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME, new EntityOwnershipListener() {
 322                 @Override
 323                 public void ownershipChanged(EntityOwnershipChange ownershipChange) {
 324                     LOG.trace("entity owner change event fired");
 325                     if (ownershipChange.hasOwner() && ownershipChange.isOwner()) {
 326                         LOG.trace("This PL is the Owner");
 327                         activateMIP();
 328                         bgpRestarted();
 329                     } else {
 330                         LOG.info("Not owner: hasOwner: {}, isOwner: {}",ownershipChange.hasOwner(),
 331                                 ownershipChange.isOwner() );
 332                     }
 333                 }
 334             });
 335         } catch (Exception e) {
 336             LOG.error("failed to register bgp entity", e);
 337         }
 338     }
 339
 340     private static final String addWarn =
 341             "Config store updated; undo with Delete if needed.";
 342     private static final String delWarn =
 343             "Config store updated; undo with Add if needed.";
 344     private static final String updWarn =
 345             "Update operation not supported; Config store updated;"
 346                     + " restore with another Update if needed.";
 347
 348     public class ConfigServerReactor
 349             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 350             implements AutoCloseable, ClusteredDataTreeChangeListener <ConfigServer> {
 351         private static final String yangObj = "config-server ";
 352
 353         public ConfigServerReactor() {
 354             super(ConfigServer.class, ConfigServerReactor.class);
 355         }
 356
 357         @Override
 358         protected synchronized void
 359         add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 360             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 361             if (ignoreClusterDcnEventForFollower()) {
 362                 return;
 363             }
 364
 365             try {
 366                 initer.await();
 367             } catch (Exception e) {
 368             }
 369             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 370             synchronized (BgpConfigurationManager.this) {
 371                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 372                         val.getPort().intValue());
 373                 if (!res) {
 374                     LOG.error(yangObj + "Add failed; " + addWarn);
 375                 }
 376             }
 377         }
 378
 379         @Override
 380         protected ConfigServerReactor getDataTreeChangeListener() {
 381             return ConfigServerReactor.this;
 382         }
 383
 384         @Override
 385         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 386             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 387         }
 388
 389         @Override
 390         protected synchronized void
 391         remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 392             LOG.trace("received bgp disconnect");
 393             if (ignoreClusterDcnEventForFollower()) {
 394                 return;
 395             }
 396             synchronized (BgpConfigurationManager.this) {
 397                 bgpRouter.disconnect();
 398             }
 399         }
 400
 401         @Override
 402         protected void update(InstanceIdentifier<ConfigServer> iid,
 403                               ConfigServer oldval, ConfigServer newval) {
 404             LOG.trace("received bgp Connection update");
 405             if (ignoreClusterDcnEventForFollower()) {
 406                 return;
 407             }
 408             LOG.error(yangObj + updWarn);
 409         }
 410
 411         @Override
 412         public void close() {
 413             try {
 414                 super.close();
 415             } catch (Exception e) {
 416                 LOG.error("ConfigServerReactor failed to close: ", e);
 417             }
 418         }
 419     }
 420
 421     private BgpRouter getClient(String yangObj) {
 422         if (bgpRouter == null) {
 423             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 424         }
 425         return bgpRouter;
 426     }
 427
 428     public class AsIdReactor
 429             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 430             implements AutoCloseable, ClusteredDataTreeChangeListener<AsId> {
 431
 432         private static final String yangObj = "as-id ";
 433
 434         public AsIdReactor() {
 435             super(AsId.class, AsIdReactor.class);
 436         }
 437
 438         @Override
 439         protected synchronized void
 440         add(InstanceIdentifier<AsId> iid, AsId val) {
 441             LOG.error("received bgp add asid {}",val);
 442             if (ignoreClusterDcnEventForFollower()) {
 443                 return;
 444             }
 445             LOG.debug("received add router config asNum {}", val.getLocalAs().intValue());
 446             synchronized (BgpConfigurationManager.this) {
 447                 BgpRouter br = getClient(yangObj);
 448                 if (br == null) {
 449                     LOG.error("no bgp router client found exiting asid add");
 450                     return;
 451                 }
 452                 int asNum = val.getLocalAs().intValue();
 453                 Ipv4Address routerId = val.getRouterId();
 454                 Boolean afb = val.isAnnounceFbit();
 455                 String rid = (routerId == null) ? "" : routerId.getValue();
 456                 int stalepathTime = (int) getStalePathtime(RESTART_DEFAULT_GR, val);
 457                 boolean announceFbit = true;
 458                 try {
 459                     br.startBgp(asNum, rid, stalepathTime, announceFbit);
 460                     if (getBgpCounters() == null) {
 461                         startBgpCountersTask();
 462                     }
 463                     if (getBgpAlarms() == null) {
 464                         startBgpAlarmsTask();
 465                     }
 466                 } catch (BgpRouterException bre) {
 467                     if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
 468                         LOG.error(yangObj + "Add requested when BGP is already active");
 469                     } else {
 470                         LOG.error(yangObj + "Add received exception: \""
 471                                 + bre + "\"; " + addWarn);
 472                     }
 473                 } catch (Exception e) {
 474                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; " + addWarn);
 475                 }
 476             }
 477         }
 478
 479         @Override
 480         protected AsIdReactor getDataTreeChangeListener() {
 481             return AsIdReactor.this;
 482         }
 483
 484         @Override
 485         protected InstanceIdentifier<AsId> getWildCardPath() {
 486             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 487         }
 488
 489         @Override
 490         protected synchronized void
 491         remove(InstanceIdentifier<AsId> iid, AsId val) {
 492             LOG.error("received delete router config asNum {}", val.getLocalAs().intValue());
 493             if (ignoreClusterDcnEventForFollower()) {
 494                 return;
 495             }
 496             synchronized (BgpConfigurationManager.this) {
 497                 BgpRouter br = getClient(yangObj);
 498                 if (br == null) {
 499                     return;
 500                 }
 501                 int asNum = val.getLocalAs().intValue();
 502                 try {
 503                     br.stopBgp(asNum);
 504                 } catch (Exception e) {
 505                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; " + delWarn);
 506                 }
 507                 if (getBgpCounters() != null) {
 508                     stopBgpCountersTask();
 509                 }
 510                 if (getBgpAlarms() != null) {
 511                     stopBgpAlarmsTask();
 512                 }
 513             }
 514         }
 515
 516         @Override
 517         protected void update(InstanceIdentifier<AsId> iid,
 518                               AsId oldval, AsId newval) {
 519             if (ignoreClusterDcnEventForFollower()) {
 520                 return;
 521             }
 522             LOG.error(yangObj + updWarn);
 523         }
 524
 525         @Override
 526         public void close() {
 527             try {
 528                 super.close();
 529             } catch (Exception e) {
 530                 e.printStackTrace();
 531             }
 532         }
 533     }
 534
 535     public class GracefulRestartReactor
 536             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 537             implements AutoCloseable, ClusteredDataTreeChangeListener<GracefulRestart> {
 538
 539         private static final String yangObj = "graceful-restart ";
 540
 541         public GracefulRestartReactor() {
 542             super(GracefulRestart.class, GracefulRestartReactor.class);
 543         }
 544
 545         @Override
 546         protected synchronized void
 547         add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 548             if (ignoreClusterDcnEventForFollower()) {
 549                 return;
 550             }
 551             synchronized (BgpConfigurationManager.this) {
 552                 BgpRouter br = getClient(yangObj);
 553                 if (br == null) {
 554                     return;
 555                 }
 556                 try {
 557                     br.addGracefulRestart(val.getStalepathTime().intValue());
 558                 } catch (Exception e) {
 559                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; " + addWarn);
 560                 }
 561             }
 562         }
 563
 564         @Override
 565         protected GracefulRestartReactor getDataTreeChangeListener() {
 566             return GracefulRestartReactor.this;
 567         }
 568
 569         @Override
 570         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 571             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 572         }
 573
 574         @Override
 575         protected synchronized void
 576         remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 577             if (ignoreClusterDcnEventForFollower()) {
 578                 return;
 579             }
 580             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 581             synchronized (BgpConfigurationManager.this) {
 582                 BgpRouter br = getClient(yangObj);
 583                 if (br == null) {
 584                     return;
 585                 }
 586                 try {
 587                     br.delGracefulRestart();
 588                 } catch (Exception e) {
 589                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 590                             + delWarn);
 591                 }
 592             }
 593         }
 594
 595         @Override
 596         protected void update(InstanceIdentifier<GracefulRestart> iid,
 597                               GracefulRestart oldval, GracefulRestart newval) {
 598             if (ignoreClusterDcnEventForFollower()) {
 599                 return;
 600             }
 601             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 602             synchronized (BgpConfigurationManager.this) {
 603                 BgpRouter br = getClient(yangObj);
 604                 if (br == null) {
 605                     return;
 606                 }
 607                 try {
 608                     br.addGracefulRestart(newval.getStalepathTime().intValue());
 609                 } catch (Exception e) {
 610                     LOG.error(yangObj + "update received exception: \"" + e + "\"; " + addWarn);
 611                 }
 612             }
 613         }
 614
 615         @Override
 616         public void close() {
 617             try {
 618                 super.close();
 619             } catch (Exception e) {
 620                 e.printStackTrace();
 621             }
 622         }
 623     }
 624
 625     public class LoggingReactor
 626             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 627             implements AutoCloseable, ClusteredDataTreeChangeListener<Logging> {
 628
 629         private static final String yangObj = "logging ";
 630
 631         public LoggingReactor() {
 632             super(Logging.class, LoggingReactor.class);
 633         }
 634
 635         @Override
 636         protected synchronized void
 637         add(InstanceIdentifier<Logging> iid, Logging val) {
 638             if (ignoreClusterDcnEventForFollower()) {
 639                 return;
 640             }
 641             synchronized (BgpConfigurationManager.this) {
 642                 BgpRouter br = getClient(yangObj);
 643                 if (br == null) {
 644                     return;
 645                 }
 646                 try {
 647                     br.setLogging(val.getFile(), val.getLevel());
 648                 } catch (Exception e) {
 649                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 650                             + addWarn);
 651                 }
 652             }
 653         }
 654
 655         @Override
 656         protected LoggingReactor getDataTreeChangeListener() {
 657             return LoggingReactor.this;
 658         }
 659
 660         @Override
 661         protected InstanceIdentifier<Logging> getWildCardPath() {
 662             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 663         }
 664
 665         @Override
 666         protected synchronized void
 667         remove(InstanceIdentifier<Logging> iid, Logging val) {
 668             if (ignoreClusterDcnEventForFollower()) {
 669                 return;
 670             }
 671             LOG.debug("received remove Logging config val {}", val.getLevel());
 672             synchronized (BgpConfigurationManager.this) {
 673                 BgpRouter br = getClient(yangObj);
 674                 if (br == null) {
 675                     return;
 676                 }
 677                 try {
 678                     br.setLogging(DEF_LOGFILE, DEF_LOGLEVEL);
 679                 } catch (Exception e) {
 680                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 681                             + delWarn);
 682                 }
 683             }
 684         }
 685
 686         @Override
 687         protected void update(InstanceIdentifier<Logging> iid,
 688                               Logging oldval, Logging newval) {
 689             if (ignoreClusterDcnEventForFollower()) {
 690                 return;
 691             }
 692             synchronized (BgpConfigurationManager.this) {
 693                 BgpRouter br = getClient(yangObj);
 694                 if (br == null) {
 695                     return;
 696                 }
 697                 try {
 698                     br.setLogging(newval.getFile(), newval.getLevel());
 699                 } catch (Exception e) {
 700                     LOG.error(yangObj + "newval received exception: \"" + e + "\"; "
 701                             + addWarn);
 702                 }
 703             }
 704         }
 705
 706         @Override
 707         public void close() {
 708             try {
 709                 super.close();
 710             } catch (Exception e) {
 711                 e.printStackTrace();
 712             }
 713         }
 714     }
 715
 716     public class NeighborsReactor
 717             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 718             implements AutoCloseable, ClusteredDataTreeChangeListener<Neighbors> {
 719
 720         private static final String yangObj = "neighbors ";
 721
 722         public NeighborsReactor() {
 723             super(Neighbors.class, NeighborsReactor.class);
 724         }
 725
 726         @Override
 727         protected synchronized void
 728         add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 729             if (ignoreClusterDcnEventForFollower()) {
 730                 return;
 731             }
 732             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 733             synchronized (BgpConfigurationManager.this) {
 734                 BgpRouter br = getClient(yangObj);
 735                 if (br == null) {
 736                     return;
 737                 }
 738                 String peerIp = val.getAddress().getValue();
 739                 int as = val.getRemoteAs().intValue();
 740                 try {
 741                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 742                     br.addNeighbor(peerIp, as);
 743
 744                 } catch (Exception e) {
 745                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 746                             + addWarn);
 747                 }
 748             }
 749         }
 750
 751         @Override
 752         protected NeighborsReactor getDataTreeChangeListener() {
 753             return NeighborsReactor.this;
 754         }
 755
 756         @Override
 757         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 758             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class);
 759         }
 760
 761         @Override
 762         protected synchronized void
 763         remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 764             if (ignoreClusterDcnEventForFollower()) {
 765                 return;
 766             }
 767             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 768             synchronized (BgpConfigurationManager.this) {
 769                 BgpRouter br = getClient(yangObj);
 770                 if (br == null) {
 771                     return;
 772                 }
 773                 String peerIp = val.getAddress().getValue();
 774                 try {
 775                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 776                     br.delNeighbor(peerIp);
 777                 } catch (Exception e) {
 778                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 779                             + delWarn);
 780                 }
 781             }
 782         }
 783
 784         @Override
 785         protected void update(InstanceIdentifier<Neighbors> iid,
 786                               Neighbors oldval, Neighbors newval) {
 787             if (ignoreClusterDcnEventForFollower()) {
 788                 return;
 789             }
 790             //purposefully nothing to do.
 791         }
 792
 793         @Override
 794         public void close() {
 795             try {
 796                 super.close();
 797             } catch (Exception e) {
 798                 e.printStackTrace();
 799             }
 800         }
 801     }
 802
 803     public class EbgpMultihopReactor
 804             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 805             implements AutoCloseable, ClusteredDataTreeChangeListener<EbgpMultihop> {
 806
 807         private static final String yangObj = "ebgp-multihop ";
 808
 809         public EbgpMultihopReactor() {
 810             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 811         }
 812
 813         @Override
 814         protected synchronized void
 815         add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 816             if (ignoreClusterDcnEventForFollower()) {
 817                 return;
 818             }
 819             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 820             synchronized (BgpConfigurationManager.this) {
 821                 BgpRouter br = getClient(yangObj);
 822                 if (br == null) {
 823                     return;
 824                 }
 825                 String peerIp = val.getPeerIp().getValue();
 826                 try {
 827                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 828                 } catch (Exception e) {
 829                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 830                             + addWarn);
 831                 }
 832             }
 833         }
 834
 835         @Override
 836         protected EbgpMultihopReactor getDataTreeChangeListener() {
 837             return EbgpMultihopReactor.this;
 838         }
 839
 840         @Override
 841         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
 842             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(EbgpMultihop.class);
 843         }
 844
 845         @Override
 846         protected synchronized void
 847         remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 848             if (ignoreClusterDcnEventForFollower()) {
 849                 return;
 850             }
 851             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 852             synchronized (BgpConfigurationManager.this) {
 853                 BgpRouter br = getClient(yangObj);
 854                 if (br == null) {
 855                     return;
 856                 }
 857                 String peerIp = val.getPeerIp().getValue();
 858                 try {
 859                     br.delEbgpMultihop(peerIp);
 860                 } catch (Exception e) {
 861                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 862                             + delWarn);
 863                 }
 864             }
 865         }
 866
 867         @Override
 868         protected void update(InstanceIdentifier<EbgpMultihop> iid,
 869                               EbgpMultihop oldval, EbgpMultihop newval) {
 870             if (ignoreClusterDcnEventForFollower()) {
 871                 return;
 872             }
 873             LOG.error(yangObj + updWarn);
 874         }
 875
 876         @Override
 877         public void close() {
 878             try {
 879                 super.close();
 880             } catch (Exception e) {
 881                 e.printStackTrace();
 882             }
 883         }
 884     }
 885
 886     public class UpdateSourceReactor
 887             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
 888             implements AutoCloseable, ClusteredDataTreeChangeListener<UpdateSource> {
 889
 890         private static final String yangObj = "update-source ";
 891
 892         public UpdateSourceReactor() {
 893             super(UpdateSource.class, UpdateSourceReactor.class);
 894         }
 895
 896         @Override
 897         protected synchronized void
 898         add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 899             if (ignoreClusterDcnEventForFollower()) {
 900                 return;
 901             }
 902             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
 903             synchronized (BgpConfigurationManager.this) {
 904                 BgpRouter br = getClient(yangObj);
 905                 if (br == null) {
 906                     return;
 907                 }
 908                 String peerIp = val.getPeerIp().getValue();
 909                 try {
 910                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
 911                 } catch (Exception e) {
 912                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 913                             + addWarn);
 914                 }
 915             }
 916         }
 917
 918         @Override
 919         protected UpdateSourceReactor getDataTreeChangeListener() {
 920             return UpdateSourceReactor.this;
 921         }
 922
 923         @Override
 924         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
 925             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(UpdateSource.class);
 926         }
 927
 928         @Override
 929         protected synchronized void
 930         remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 931             if (ignoreClusterDcnEventForFollower()) {
 932                 return;
 933             }
 934             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
 935             synchronized (BgpConfigurationManager.this) {
 936                 BgpRouter br = getClient(yangObj);
 937                 if (br == null) {
 938                     return;
 939                 }
 940                 String peerIp = val.getPeerIp().getValue();
 941                 try {
 942                     br.delUpdateSource(peerIp);
 943                 } catch (Exception e) {
 944                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 945                             + delWarn);
 946                 }
 947             }
 948         }
 949
 950         @Override
 951         protected void update(InstanceIdentifier<UpdateSource> iid,
 952                               UpdateSource oldval, UpdateSource newval) {
 953             if (ignoreClusterDcnEventForFollower()) {
 954                 return;
 955             }
 956             LOG.error(yangObj + updWarn);
 957         }
 958
 959         @Override
 960         public void close() {
 961             try {
 962                 super.close();
 963             } catch (Exception e) {
 964                 e.printStackTrace();
 965             }
 966         }
 967     }
 968
 969     public class AddressFamiliesReactor
 970             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
 971             implements AutoCloseable, ClusteredDataTreeChangeListener<AddressFamilies> {
 972
 973         private static final String yangObj = "address-families ";
 974
 975         public AddressFamiliesReactor() {
 976             super(AddressFamilies.class, AddressFamiliesReactor.class);
 977         }
 978
 979         @Override
 980         protected synchronized void
 981         add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
 982             if (ignoreClusterDcnEventForFollower()) {
 983                 return;
 984             }
 985             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
 986             synchronized (BgpConfigurationManager.this) {
 987                 BgpRouter br = getClient(yangObj);
 988                 if (br == null) {
 989                     return;
 990                 }
 991                 String peerIp = val.getPeerIp().getValue();
 992                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
 993                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
 994                 try {
 995                     br.addAddressFamily(peerIp, afi, safi);
 996                 } catch (Exception e) {
 997                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 998                             + addWarn);
 999                 }
1000             }
1001         }
1002
1003         @Override
1004         protected AddressFamiliesReactor getDataTreeChangeListener() {
1005             return AddressFamiliesReactor.this;
1006         }
1007
1008         @Override
1009         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
1010             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(AddressFamilies.class);
1011         }
1012
1013         @Override
1014         protected synchronized void
1015         remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1016             if (ignoreClusterDcnEventForFollower()) {
1017                 return;
1018             }
1019             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
1020             synchronized (BgpConfigurationManager.this) {
1021                 BgpRouter br = getClient(yangObj);
1022                 if (br == null) {
1023                     return;
1024                 }
1025                 String peerIp = val.getPeerIp().getValue();
1026                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1027                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1028                 try {
1029                     br.delAddressFamily(peerIp, afi, safi);
1030                 } catch (Exception e) {
1031                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
1032                             + delWarn);
1033                 }
1034             }
1035         }
1036
1037         @Override
1038         protected void update(InstanceIdentifier<AddressFamilies> iid,
1039                               AddressFamilies oldval, AddressFamilies newval) {
1040             if (ignoreClusterDcnEventForFollower()) {
1041                 return;
1042             }
1043             LOG.error(yangObj + updWarn);
1044         }
1045
1046         @Override
1047         public void close() {
1048             try {
1049                 super.close();
1050             } catch (Exception e) {
1051                 e.printStackTrace();
1052             }
1053         }
1054     }
1055
1056     public class NetworksReactor
1057             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
1058             implements AutoCloseable, ClusteredDataTreeChangeListener<Networks> {
1059
1060         private static final String yangObj = "networks ";
1061
1062         public NetworksReactor() {
1063             super(Networks.class, NetworksReactor.class);
1064         }
1065         @Override
1066         public NetworksReactor getDataTreeChangeListener() {
1067             return NetworksReactor.this;
1068         }
1069
1070         @Override
1071         protected synchronized void
1072         add(InstanceIdentifier<Networks> iid, Networks val) {
1073             if (ignoreClusterDcnEventForFollower()) {
1074                 return;
1075             }
1076             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1077             synchronized (BgpConfigurationManager.this) {
1078                 BgpRouter br = getClient(yangObj);
1079                 if (br == null) {
1080                     return;
1081                 }
1082                 String rd = val.getRd();
1083                 String pfxlen = val.getPrefixLen();
1084                 String nh = val.getNexthop().getValue();
1085                 Long label = val.getLabel();
1086                 int lbl = (label == null) ? qbgpConstants.LBL_NO_LABEL
1087                         : label.intValue();
1088                 try {
1089                     br.addPrefix(rd, pfxlen, nh, lbl);
1090                 } catch (Exception e) {
1091                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; " + addWarn);
1092                 }
1093             }
1094         }
1095
1096         @Override
1097         protected InstanceIdentifier<Networks> getWildCardPath() {
1098             return InstanceIdentifier.create(Bgp.class).child(Networks.class);
1099         }
1100
1101         @Override
1102         protected synchronized void
1103         remove(InstanceIdentifier<Networks> iid, Networks val) {
1104             if (ignoreClusterDcnEventForFollower()) {
1105                 return;
1106             }
1107             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1108             synchronized (BgpConfigurationManager.this) {
1109                 BgpRouter br = getClient(yangObj);
1110                 if (br == null) {
1111                     return;
1112                 }
1113                 String rd = val.getRd();
1114                 String pfxlen = val.getPrefixLen();
1115                 Long label = val.getLabel();
1116                 int lbl = (label == null) ? 0 : label.intValue();
1117                 if (rd == null && lbl > 0) {
1118                     //LU prefix is being deleted.
1119                     rd = Integer.toString(lbl);
1120                 }
1121                 try {
1122                     br.delPrefix(rd, pfxlen);
1123                 } catch (Exception e) {
1124                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
1125                             + delWarn);
1126                 }
1127             }
1128         }
1129
1130         @Override
1131         protected void update(final InstanceIdentifier<Networks> iid,
1132                               final Networks oldval, final Networks newval) {
1133             if (ignoreClusterDcnEventForFollower()) {
1134                 return;
1135             }
1136             if(oldval.equals(newval)){
1137                 //Update: OLD and New values are same, no need to trigger remove/add.
1138                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1139                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1140                 return;
1141             }
1142             LOG.debug("received update networks config val {}", newval.getPrefixLen());
1143             remove(iid, oldval);
1144             timer.schedule(new TimerTask() {
1145                 @Override
1146                 public void run() {
1147                     add(iid, newval);
1148                 }
1149             }, Integer.getInteger("bgp.nexthop.update.delay.in.secs", 5) * 1000);
1150         }
1151
1152         @Override
1153         public void close() {
1154             try {
1155                 super.close();
1156             } catch (Exception e) {
1157                 e.printStackTrace();
1158             }
1159         }
1160     }
1161
1162     static Timer timer = new Timer();
1163
1164     public class VrfsReactor
1165             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1166             implements AutoCloseable, ClusteredDataTreeChangeListener<Vrfs> {
1167
1168         private static final String yangObj = "vrfs ";
1169
1170         public VrfsReactor() {
1171             super(Vrfs.class, VrfsReactor.class);
1172         }
1173
1174         @Override
1175         protected synchronized void
1176         add(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1177             if (ignoreClusterDcnEventForFollower()) {
1178                 return;
1179             }
1180             LOG.debug("received add Vrfs config val {}", val.getRd());
1181             synchronized (BgpConfigurationManager.this) {
1182                 BgpRouter br = getClient(yangObj);
1183                 if (br == null) {
1184                     return;
1185                 }
1186                 try {
1187                     br.addVrf(val.getRd(), val.getImportRts(),
1188                             val.getExportRts());
1189                 } catch (Exception e) {
1190                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
1191                             + addWarn);
1192                 }
1193             }
1194         }
1195
1196         @Override
1197         protected VrfsReactor getDataTreeChangeListener() {
1198             return VrfsReactor.this;
1199         }
1200
1201         @Override
1202         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1203             return InstanceIdentifier.create(Bgp.class).child(Vrfs.class);
1204         }
1205
1206         @Override
1207         protected synchronized void
1208         remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1209             if (ignoreClusterDcnEventForFollower()) {
1210                 return;
1211             }
1212             LOG.debug("received remove Vrfs config val {}", val.getRd());
1213             synchronized (BgpConfigurationManager.this) {
1214                 BgpRouter br = getClient(yangObj);
1215                 if (br == null) {
1216                     return;
1217                 }
1218                 try {
1219                     br.delVrf(val.getRd());
1220                 } catch (Exception e) {
1221                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
1222                             + delWarn);
1223                 }
1224             }
1225         }
1226
1227         @Override
1228         protected void update(InstanceIdentifier<Vrfs> iid,
1229                               Vrfs oldval, Vrfs newval) {
1230             if (ignoreClusterDcnEventForFollower()) {
1231                 return;
1232             }
1233             LOG.debug("VRFS: Update getting triggered for VRFS rd {}", oldval.getRd());
1234             LOG.error(yangObj + updWarn);
1235         }
1236
1237         @Override
1238         public void close() {
1239             try {
1240                 super.close();
1241             } catch (Exception e) {
1242                 e.printStackTrace();
1243             }
1244         }
1245     }
1246
1247     Future lastCleanupJob;
1248     Future lastReplayJobFt = null;
1249     protected void activateMIP() {
1250         try {
1251             LOG.trace("BgpReactor: Executing MIP Activate command");
1252             Process process_bgp = Runtime.getRuntime().exec("cluster ip -a sdnc_bgp_mip");
1253             Process process_os = Runtime.getRuntime().exec("cluster ip -a sdnc_os_mip");
1254             LOG.trace("bgpMIP Activated");
1255
1256         } catch (IOException io) {
1257             //LOG.error("IO Exception got while activating mip:  ", io);
1258         } catch (Exception e) {
1259             //LOG.error("Exception got while activating mip: ", e);
1260         }
1261     }
1262
1263     AtomicBoolean started = new AtomicBoolean(false);
1264
1265     public class BgpReactor
1266             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1267             implements AutoCloseable, ClusteredDataTreeChangeListener<Bgp> {
1268
1269         private static final String yangObj = "Bgp ";
1270
1271         public BgpReactor() {
1272             super(Bgp.class, BgpReactor.class);
1273         }
1274
1275
1276         @Override
1277         protected synchronized void
1278         add(InstanceIdentifier<Bgp> iid, Bgp val) {
1279             LOG.error("received add Bgp config replaying the config");
1280
1281             try {
1282                 initer.await();
1283             } catch (Exception e) {
1284             }
1285             synchronized (BgpConfigurationManager.this) {
1286                 config = val;
1287                 if (ignoreClusterDcnEventForFollower()) {
1288                     return;
1289                 }
1290                 activateMIP();
1291                 if (isIpAvailable(odlThriftIp)) {
1292                     bgpRestarted();
1293                 } else {
1294                     ipActivationCheckTimer.scheduleAtFixedRate(new TimerTask() {
1295                         @Override
1296                         public void run() {
1297                             if (isIpAvailable(odlThriftIp)) {
1298                                 bgpRestarted();
1299                                 ipActivationCheckTimer.cancel();
1300                             } else {
1301                                 LOG.trace("waiting for odlThriftIP: {} to be present", odlThriftIp);
1302                             }
1303                         }
1304                     }, 10000L, 10000L);
1305                 }
1306             }
1307         }
1308
1309         @Override
1310         protected BgpReactor getDataTreeChangeListener() {
1311             return BgpReactor.this;
1312         }
1313
1314         @Override
1315         protected InstanceIdentifier<Bgp> getWildCardPath() {
1316             return InstanceIdentifier.create(Bgp.class);
1317         }
1318
1319         @Override
1320         protected synchronized void
1321         remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1322             if (ignoreClusterDcnEventForFollower()) {
1323                 return;
1324             }
1325             LOG.debug("received remove Bgp config");
1326             synchronized (BgpConfigurationManager.this) {
1327                 config = null;
1328             }
1329         }
1330
1331         @Override
1332         protected void update(InstanceIdentifier<Bgp> iid,
1333                               Bgp oldval, Bgp newval) {
1334             if (ignoreClusterDcnEventForFollower()) {
1335                 return;
1336             }
1337             synchronized (BgpConfigurationManager.this) {
1338                 config = newval;
1339             }
1340         }
1341
1342         @Override
1343         public void close() {
1344             try {
1345                 super.close();
1346             } catch (Exception e) {
1347                 e.printStackTrace();
1348             }
1349         }
1350     }
1351
1352     public String readThriftIpForCommunication( String mipAddr) {
1353         File f = new File(CLUSTER_CONF_FILE);
1354         if (!f.exists()) {
1355             return  DEF_CHOST;
1356         }
1357         BufferedReader br = null;
1358         try {
1359             br = new BufferedReader(new InputStreamReader(
1360                     new FileInputStream(f)));
1361             String line = br.readLine();
1362             while (line != null) {
1363                 if (line.contains(mipAddr)) {
1364                     line = line.trim();
1365                     return line.substring(line.lastIndexOf(" ") + 1);
1366                 }
1367                 line = br.readLine();
1368             }
1369         } catch (Exception e) {
1370         } finally {
1371             try {
1372                 br.close();
1373             } catch (Exception ignore) {
1374             }
1375         }
1376         return DEF_CHOST;
1377     }
1378
1379     public boolean isIpAvailable(String odlip) {
1380
1381         try {
1382             if (odlip != null) {
1383                 if ("127.0.0.1".equals(odlip)) {
1384                     return true;
1385                 }
1386                 Enumeration e = NetworkInterface.getNetworkInterfaces();
1387                 while (e.hasMoreElements()) {
1388                     NetworkInterface n = (NetworkInterface) e.nextElement();
1389                     Enumeration ee = n.getInetAddresses();
1390                     while (ee.hasMoreElements()) {
1391                         InetAddress i = (InetAddress) ee.nextElement();
1392                         if (odlip.equals(i.getHostAddress())) {
1393                             return true;
1394                         }
1395                         ;
1396                     }
1397                 }
1398             }
1399         } catch (Exception e) {
1400         }
1401         return false;
1402     }
1403
1404     public static long getStalePathtime(int defValue, AsId as_num) {
1405         long spt = 0;
1406         try {
1407             spt = getConfig().getGracefulRestart().getStalepathTime();
1408         } catch (Exception e) {
1409             try {
1410                 spt = as_num.getStalepathTime();
1411                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1412             } catch (Exception ignore) {
1413                 LOG.trace("BGP AS id is not set using graceful");
1414                 spt = defValue;
1415             }
1416         }
1417         if (spt == 0) {
1418             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1419             spt = defValue;
1420         }
1421         return spt;
1422     }
1423
1424     public synchronized void bgpRestarted() {
1425         /*
1426          * If there a thread which in the process of stale cleanup, cancel it
1427          * and start a new thread (to avoid processing same again).
1428          */
1429         if (previousReplayJobInProgress()) {
1430             cancelPreviousReplayJob();
1431         }
1432         Runnable task = new Runnable() {
1433             @Override
1434             public void run() {
1435                 try {
1436                     LOG.info("running bgp replay task ");
1437                     if (get() == null) {
1438                         String host = getConfigHost();
1439                         int port = getConfigPort();
1440                         LOG.info("connecting  to bgp host {} ", host);
1441
1442                         boolean res = bgpRouter.connect(host, port);
1443                         LOG.info("no config to push in bgp replay task ");
1444                         return;
1445                     }
1446                     setStaleStartTime(System.currentTimeMillis());
1447                     LOG.info("started creating stale fibDSWriter  map ");
1448                     createStaleFibMap();
1449                     setStaleEndTime(System.currentTimeMillis());
1450                     LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime()- getStaleStartTime());
1451                     LOG.info("started bgp config replay ");
1452                     setCfgReplayStartTime(System.currentTimeMillis());
1453                     replay();
1454                     setCfgReplayEndTime(System.currentTimeMillis());
1455                     LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1456                     long route_sync_time = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1457                     Thread.sleep(route_sync_time * 1000L);
1458                     setStaleCleanupTime(route_sync_time);
1459                     new RouteCleanup().call();
1460                 } catch (Exception eCancel) {
1461                     LOG.error("Stale Cleanup Task Cancelled", eCancel);
1462                 }
1463             }
1464         };
1465         lastReplayJobFt = executor.submit(task);
1466     }
1467
1468     private boolean previousReplayJobInProgress() {
1469         return lastReplayJobFt != null && !lastReplayJobFt.isDone();
1470     }
1471
1472     private void cancelPreviousReplayJob() {
1473         try {
1474             LOG.error("cancelling already running bgp replay task");
1475             lastReplayJobFt.cancel(true);
1476             lastReplayJobFt = null;
1477             Thread.sleep(2000);
1478         } catch (Throwable e) {
1479             LOG.error("Failed to cancel previous replay job ",e);
1480         }
1481     }
1482
1483     private static void doRouteSync() {
1484         BgpSyncHandle bsh = BgpSyncHandle.getInstance();
1485         LOG.error("Starting BGP route sync");
1486         try {
1487             bgpRouter.initRibSync(bsh);
1488         } catch (Exception e) {
1489             LOG.error("Route sync aborted, exception when initializing: " + e);
1490             return;
1491         }
1492         while (bsh.getState() != bsh.DONE) {
1493             Routes routes = null;
1494             try {
1495                 routes = bgpRouter.doRibSync(bsh);
1496             } catch (Exception e) {
1497                 LOG.error("Route sync aborted, exception when syncing: " + e);
1498                 return;
1499             }
1500             Iterator<Update> updates = routes.getUpdatesIterator();
1501             while (updates.hasNext()) {
1502                 Update u = updates.next();
1503                 Map<String, Map<String, String>> stale_fib_rd_map = BgpConfigurationManager.getStaledFibEntriesMap();
1504                 String rd = u.getRd();
1505                 String nexthop = u.getNexthop();
1506                 int label = u.getLabel();
1507                 String prefix = u.getPrefix();
1508                 int plen = u.getPrefixlen();
1509                 onUpdatePushRoute(rd, prefix, plen, nexthop, label);
1510             }
1511         }
1512         try {
1513             LOG.error("Ending BGP route-sync");
1514             bgpRouter.endRibSync(bsh);
1515         } catch (Exception e) {
1516         }
1517     }
1518
1519     /* onUpdatePushRoute
1520      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1521      *  - Entry compare shall include NextHop, Label.
1522      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1523      *  - If entry not found, add to FIB Config DS.
1524      *  - If entry found, but either Label/NextHop doesn't match.
1525      *      - Update FIB Config DS with modified values.
1526      *      - delete from Stale Map.
1527      */
1528     public static void onUpdatePushRoute(String rd, String prefix, int plen, String nextHop, int label) {
1529         Map<String, Map<String, String>> stale_fib_rd_map = BgpConfigurationManager.getStaledFibEntriesMap();
1530         boolean addroute = false;
1531         if (!stale_fib_rd_map.isEmpty()) {
1532             // restart Scenario, as MAP is not empty.
1533             Map<String, String> map = stale_fib_rd_map.get(rd);
1534             if (map != null) {
1535                 String nexthoplabel = map.get(prefix + "/" + plen);
1536                 if (null == nexthoplabel) {
1537                     // New Entry, which happened to be added during restart.
1538                     addroute = true;
1539                 } else {
1540                     map.remove(prefix + "/" + plen);
1541                     if (isRouteModified(nextHop, label, nexthoplabel)) {
1542                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1543                         // Existing entry, where in Nexthop/Label got modified during restart
1544                         addroute = true;
1545                     }
1546                 }
1547             }
1548         } else {
1549             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1550             addroute = true;
1551         }
1552         if (addroute) {
1553             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1554             fibDSWriter.addFibEntryToDS(rd, prefix + "/" + plen, Arrays.asList(nextHop), label, RouteOrigin.BGP);
1555             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1556         }
1557     }
1558
1559     private static boolean isRouteModified(String nexthop, int label, String nexthoplabel) {
1560         return !nexthoplabel.isEmpty() && !nexthoplabel.equals(nexthop + "/" + label);
1561     }
1562
1563     static private void replayNbrConfig(List<Neighbors> n, BgpRouter br) {
1564         for (Neighbors nbr : n) {
1565             try {
1566                 br.addNeighbor(nbr.getAddress().getValue(),
1567                         nbr.getRemoteAs().intValue());
1568                 //itmProvider.buildTunnelsToDCGW(new IpAddress(nbr.getAddress().getValue().toCharArray()));
1569             } catch (Exception e) {
1570                 LOG.error("Replay:addNbr() received exception: \"" + e + "\"");
1571                 continue;
1572             }
1573             EbgpMultihop en = nbr.getEbgpMultihop();
1574             if (en != null) {
1575                 try {
1576                     br.addEbgpMultihop(en.getPeerIp().getValue(),
1577                             en.getNhops().intValue());
1578                 } catch (Exception e) {
1579                     LOG.error("Replay:addEBgp() received exception: \"" + e + "\"");
1580                 }
1581             }
1582             UpdateSource us = nbr.getUpdateSource();
1583             if (us != null) {
1584                 try {
1585                     br.addUpdateSource(us.getPeerIp().getValue(),
1586                             us.getSourceIp().getValue());
1587                 } catch (Exception e) {
1588                     LOG.error("Replay:addUS() received exception: \"" + e + "\"");
1589                 }
1590             }
1591             List<AddressFamilies> afs = nbr.getAddressFamilies();
1592             if (afs != null) {
1593                 for (AddressFamilies af : afs) {
1594                     af_afi afi = af_afi.findByValue(af.getAfi().intValue());
1595                     af_safi safi = af_safi.findByValue(af.getSafi().intValue());
1596                     try {
1597                         br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
1598                     } catch (Exception e) {
1599                         LOG.error("Replay:addAf() received exception: \"" + e + "\"");
1600                     }
1601                 }
1602             }
1603         }
1604     }
1605
1606     public static String getConfigHost() {
1607         if (config == null) {
1608             return cHostStartup;
1609         }
1610         ConfigServer ts = config.getConfigServer();
1611         return (ts == null ? cHostStartup : ts.getHost().getValue());
1612     }
1613
1614     public static int getConfigPort() {
1615         if (config == null) {
1616             return Integer.parseInt(cPortStartup);
1617         }
1618         ConfigServer ts = config.getConfigServer();
1619         return (ts == null ? Integer.parseInt(cPortStartup) :
1620                 ts.getPort().intValue());
1621     }
1622
1623     public static Bgp getConfig() {
1624         AtomicInteger bgpDSretryCount = new AtomicInteger(dsRetryCoount);
1625         while (0  != bgpDSretryCount.decrementAndGet()) {
1626             try {
1627                 Optional<Bgp> optional = BgpUtil.read(dataBroker,
1628                                          LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(Bgp.class));
1629                 if (optional.isPresent()) {
1630                     return optional.get();
1631                 } else {
1632                     return null;
1633                 }
1634             } catch (Exception e) {
1635                 //Config DS may not be up, so sleep for 1 second and retry
1636                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
1637                 try {
1638                     Thread.sleep(waitTimeBetweenEachTryMillis);
1639                 }catch (Exception timerEx){
1640                     LOG.debug("waitTimeBetweenEachTryMillis, Timer got interrupted while waiting for" +
1641                             "config DS availability", timerEx);
1642                 }
1643             }
1644          }
1645          LOG.error("failed to get bgp config");
1646          return null;
1647      }
1648
1649     public synchronized void replay() {
1650         synchronized (bgpConfigurationManager) {
1651             String host = getConfigHost();
1652             int port = getConfigPort();
1653             LOG.error("connecting  to bgp host {} ", host);
1654
1655             boolean res = bgpRouter.connect(host, port);
1656             if (!res) {
1657                 String msg = "Cannot connect to BGP config server at " + host + ":" + port;
1658                 if (config != null) {
1659                     msg += "; Configuration Replay aborted";
1660                 }
1661                 LOG.error(msg);
1662                 return;
1663             }
1664             config = getConfig();
1665             if (config == null) {
1666                 LOG.error("bgp config is empty nothing to push to bgp");
1667                 return;
1668             }
1669             BgpRouter br = bgpRouter;
1670             AsId a = config.getAsId();
1671             if (a == null) {
1672                 return;
1673             }
1674             int asNum = a.getLocalAs().intValue();
1675             Ipv4Address routerId = a.getRouterId();
1676             Long spt = a.getStalepathTime();
1677             Boolean afb = a.isAnnounceFbit();
1678             String rid = (routerId == null) ? "" : routerId.getValue();
1679             int stalepathTime = (int) getStalePathtime(0, config.getAsId());
1680             boolean announceFbit = true;
1681             try {
1682                 br.startBgp(asNum, rid, stalepathTime, announceFbit);
1683             } catch (BgpRouterException bre) {
1684                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
1685                     doRouteSync();
1686                 } else {
1687                     LOG.error("Replay: startBgp() received exception: \""
1688                             + bre + "\"; " + addWarn);
1689                 }
1690             } catch (Exception e) {
1691                 //not unusual. We may have restarted & BGP is already on
1692                 LOG.error("Replay:startBgp() received exception: \"" + e + "\"");
1693             }
1694
1695             if (getBgpCounters() == null) {
1696                 startBgpCountersTask();
1697             }
1698
1699             if (getBgpAlarms() == null) {
1700                 startBgpAlarmsTask();
1701             }
1702
1703             Logging l = config.getLogging();
1704             if (l != null) {
1705                 try {
1706                     br.setLogging(l.getFile(), l.getLevel());
1707                 } catch (Exception e) {
1708                     LOG.error("Replay:setLogging() received exception: \"" + e + "\"");
1709                 }
1710             }
1711
1712             GracefulRestart g = config.getGracefulRestart();
1713             if (g != null) {
1714                 try {
1715                     br.addGracefulRestart(g.getStalepathTime().intValue());
1716                 } catch (Exception e) {
1717                     LOG.error("Replay:addGr() received exception: \"" + e + "\"");
1718                 }
1719             }
1720
1721             List<Vrfs> v = config.getVrfs();
1722             if (v != null) {
1723                 for (Vrfs vrf : v) {
1724                     try {
1725                         br.addVrf(vrf.getRd(), vrf.getImportRts(),
1726                                 vrf.getExportRts());
1727                     } catch (Exception e) {
1728                         LOG.error("Replay:addVrf() received exception: \"" + e + "\"");
1729                     }
1730                 }
1731             }
1732
1733             List<Networks> ln = config.getNetworks();
1734             if (ln != null) {
1735                 for (Networks net : ln) {
1736                     String rd = net.getRd();
1737                     String pfxlen = net.getPrefixLen();
1738                     String nh = net.getNexthop().getValue();
1739                     Long label = net.getLabel();
1740                     int lbl = (label == null) ? 0 : label.intValue();
1741                     if (rd == null && lbl > 0) {
1742                         //LU prefix is being deleted.
1743                         rd = Integer.toString(lbl);
1744                     }
1745                     try {
1746                         br.addPrefix(rd, pfxlen, nh, lbl);
1747                     } catch (Exception e) {
1748                         LOG.error("Replay:addPfx() received exception: \"" + e + "\"");
1749                     }
1750                 }
1751             }
1752             List<Neighbors> n = config.getNeighbors();
1753             if (n != null) {
1754                 LOG.error("configuring existing Neighbors present for replay total neighbors {}", n.size());
1755                 replayNbrConfig(n, br);
1756             } else {
1757                 LOG.error("no Neighbors present for replay config ");
1758             }
1759         }
1760     }
1761
1762     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
1763         BgpUtil.update(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1764     }
1765
1766     private <T extends DataObject> void asyncWrite(InstanceIdentifier<T> iid, T dto) {
1767         BgpUtil.write(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1768     }
1769
1770     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
1771         BgpUtil.delete(dataBroker, LogicalDatastoreType.CONFIGURATION, iid);
1772     }
1773
1774     public synchronized void
1775     startConfig(String bgpHost, int thriftPort) {
1776         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
1777                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
1778         InstanceIdentifier<ConfigServer> iid = iib.build();
1779         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
1780         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
1781                 .setPort((long) thriftPort).build();
1782         update(iid, dto);
1783     }
1784
1785     public synchronized void
1786     startBgp(int as, String routerId, int spt, boolean fbit) {
1787         Long localAs = (long) as;
1788         Ipv4Address rid = (routerId == null) ?
1789                 null : new Ipv4Address(routerId);
1790         Long staleTime = (long) spt;
1791         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
1792                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
1793         InstanceIdentifier<AsId> iid = iib.build();
1794         AsId dto = new AsIdBuilder().setLocalAs(localAs)
1795                 .setRouterId(rid)
1796                 .setStalepathTime(staleTime)
1797                 .setAnnounceFbit(fbit).build();
1798         update(iid, dto);
1799     }
1800
1801     public synchronized void
1802     addLogging(String fileName, String logLevel) {
1803         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
1804                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
1805         InstanceIdentifier<Logging> iid = iib.build();
1806         Logging dto = new LoggingBuilder().setFile(fileName)
1807                 .setLevel(logLevel).build();
1808         update(iid, dto);
1809     }
1810
1811     public synchronized void
1812     addGracefulRestart(int staleTime) {
1813         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
1814                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
1815         InstanceIdentifier<GracefulRestart> iid = iib.build();
1816         GracefulRestart dto = new GracefulRestartBuilder()
1817                 .setStalepathTime((long) staleTime).build();
1818         update(iid, dto);
1819     }
1820
1821     public synchronized void
1822     addNeighbor(String nbrIp, int remoteAs) {
1823         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1824         Long rAs = (long) remoteAs;
1825         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
1826                 InstanceIdentifier.builder(Bgp.class)
1827                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
1828         InstanceIdentifier<Neighbors> iid = iib.build();
1829         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
1830                 .setRemoteAs(rAs).build();
1831         update(iid, dto);
1832     }
1833
1834     public synchronized void
1835     addUpdateSource(String nbrIp, String srcIp) {
1836         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1837         Ipv4Address srcAddr = new Ipv4Address(srcIp);
1838         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
1839                 InstanceIdentifier.builder(Bgp.class)
1840                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1841                         .child(UpdateSource.class);
1842         InstanceIdentifier<UpdateSource> iid = iib.build();
1843         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
1844                 .setSourceIp(srcAddr).build();
1845         update(iid, dto);
1846     }
1847
1848     public synchronized void
1849     addEbgpMultihop(String nbrIp, int nHops) {
1850         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1851         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
1852                 InstanceIdentifier.builder(Bgp.class)
1853                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1854                         .child(EbgpMultihop.class);
1855         InstanceIdentifier<EbgpMultihop> iid = iib.build();
1856         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
1857                 .setNhops((long) nHops).build();
1858         update(iid, dto);
1859     }
1860
1861     public synchronized void
1862     addAddressFamily(String nbrIp, int afi, int safi) {
1863         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1864         Long a = (long) afi;
1865         Long sa = (long) safi;
1866         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
1867                 InstanceIdentifier.builder(Bgp.class)
1868                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1869                         .child(AddressFamilies.class, new AddressFamiliesKey(a, sa));
1870         InstanceIdentifier<AddressFamilies> iid = iib.build();
1871         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
1872                 .setAfi(a).setSafi(sa).build();
1873         update(iid, dto);
1874     }
1875
1876     public synchronized void
1877     addPrefix(String rd, String pfx, List<String> nhList, int lbl) {
1878         for (String nh : nhList) {
1879             Ipv4Address nexthop = new Ipv4Address(nh);
1880             Long label = (long) lbl;
1881             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
1882                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
1883             Networks dto = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
1884                                                 .setLabel(label).build();
1885             update(iid, dto);
1886         }
1887     }
1888
1889     public synchronized void
1890     addVrf(String rd, List<String> irts, List<String> erts) {
1891         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
1892                 InstanceIdentifier.builder(Bgp.class)
1893                         .child(Vrfs.class, new VrfsKey(rd));
1894         InstanceIdentifier<Vrfs> iid = iib.build();
1895         Vrfs dto = new VrfsBuilder().setRd(rd)
1896                 .setImportRts(irts)
1897                 .setExportRts(erts).build();
1898         BgpUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1899     }
1900
1901     public synchronized void stopConfig() {
1902         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
1903                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
1904         InstanceIdentifier<ConfigServer> iid = iib.build();
1905         delete(iid);
1906     }
1907
1908     public synchronized void stopBgp() {
1909         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
1910                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
1911         InstanceIdentifier<AsId> iid = iib.build();
1912         delete(iid);
1913     }
1914
1915     public synchronized void delLogging() {
1916         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
1917                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
1918         InstanceIdentifier<Logging> iid = iib.build();
1919         delete(iid);
1920     }
1921
1922     public synchronized void delGracefulRestart() {
1923         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
1924                 InstanceIdentifier.builder(Bgp.class)
1925                         .child(GracefulRestart.class);
1926         InstanceIdentifier<GracefulRestart> iid = iib.build();
1927         delete(iid);
1928     }
1929
1930     public synchronized void delNeighbor(String nbrIp) {
1931         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1932         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
1933                 InstanceIdentifier.builder(Bgp.class)
1934                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
1935         InstanceIdentifier<Neighbors> iid = iib.build();
1936         delete(iid);
1937     }
1938
1939     public synchronized void delUpdateSource(String nbrIp) {
1940         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1941         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
1942                 InstanceIdentifier.builder(Bgp.class)
1943                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1944                         .child(UpdateSource.class);
1945         InstanceIdentifier<UpdateSource> iid = iib.build();
1946         delete(iid);
1947     }
1948
1949     public synchronized void delEbgpMultihop(String nbrIp) {
1950         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1951         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
1952                 InstanceIdentifier.builder(Bgp.class)
1953                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1954                         .child(EbgpMultihop.class);
1955         InstanceIdentifier<EbgpMultihop> iid = iib.build();
1956         delete(iid);
1957     }
1958
1959     public synchronized void
1960     delAddressFamily(String nbrIp, int afi, int safi) {
1961         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1962         Long a = (long) afi;
1963         Long sa = (long) safi;
1964         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
1965                 InstanceIdentifier.builder(Bgp.class)
1966                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1967                         .child(AddressFamilies.class, new AddressFamiliesKey(a, sa));
1968         InstanceIdentifier<AddressFamilies> iid = iib.build();
1969         delete(iid);
1970     }
1971
1972     public synchronized void delPrefix(String rd, String pfx) {
1973         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
1974                 InstanceIdentifier.builder(Bgp.class)
1975                         .child(Networks.class, new NetworksKey(pfx, rd));
1976         InstanceIdentifier<Networks> iid = iib.build();
1977         delete(iid);
1978     }
1979
1980     public synchronized void delVrf(String rd) {
1981         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
1982                 InstanceIdentifier.builder(Bgp.class)
1983                         .child(Vrfs.class, new VrfsKey(rd));
1984         InstanceIdentifier<Vrfs> iid = iib.build();
1985         delete(iid);
1986     }
1987
1988     static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
1989
1990     /*
1991     * Remove Stale Marked Routes after timer expiry.
1992     */
1993     class RouteCleanup implements Callable<Integer> {
1994
1995         @Override
1996         public Integer call() {
1997             totalCleared = 0;
1998             try {
1999                 if (staledFibEntriesMap.isEmpty()) {
2000                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
2001                 } else {
2002                     for (String rd : staledFibEntriesMap.keySet()) {
2003                         if (Thread.interrupted()) {
2004                             return 0;
2005                         }
2006                         Map<String, String> map = staledFibEntriesMap.get(rd);
2007                         if (map != null) {
2008                             for (String prefix : map.keySet()) {
2009                                 if (Thread.interrupted()) {
2010                                     return 0;
2011                                 }
2012                                 try {
2013                                     totalCleared++;
2014                                     LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}" + rd.toString() + prefix);
2015                                     fibDSWriter.removeFibEntryFromDS(rd, prefix);
2016                                 } catch (Exception e) {
2017                                     LOG.error("BGP: RouteCleanup deletePrefix failed rd:{}, prefix{}" + rd.toString() + prefix);
2018                                 }
2019                             }
2020                         }
2021                     }
2022                 }
2023             } catch (Exception e) {
2024                 LOG.error("Cleanup Thread Got interrupted, Failed to cleanup stale routes ", e);
2025             } finally {
2026                 staledFibEntriesMap.clear();
2027             }
2028             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
2029             return 0;
2030         }
2031     }
2032
2033     /*
2034      * BGP restart scenario, ODL-BGP manager was/is running.
2035      * On re-sync notification, Get a copy of FIB database.
2036      */
2037     public static void createStaleFibMap() {
2038         totalStaledCount = 0;
2039         try {
2040             /*
2041             * at the time Stale FIB creation, Wait till all PENDING write transaction
2042              * to complete (or)wait for max timeout value of STALE_FIB_WAIT Seconds.
2043              */
2044             int retry = STALE_FIB_WAIT;
2045             while ((BgpUtil.getGetPendingWrTransaction() != 0) && (retry > 0)) {
2046                 Thread.sleep(1000);
2047                 retry--;
2048                 if (retry == 0) {
2049                     LOG.error("TimeOut occured {} seconds, in waiting stale fibDSWriter create", STALE_FIB_WAIT);
2050                 }
2051             }
2052             staledFibEntriesMap.clear();
2053             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2054             DataBroker db = BgpUtil.getBroker();
2055             if (db == null) {
2056                 LOG.error("Couldn't find BgpUtil dataBroker while creating createStaleFibMap");
2057                 return;
2058             }
2059
2060             Optional<FibEntries> fibEntries = BgpUtil.read(BgpUtil.getBroker(),
2061                     LogicalDatastoreType.CONFIGURATION, id);
2062             if (fibEntries.isPresent()) {
2063                 List<VrfTables> stale_vrfTables = fibEntries.get().getVrfTables();
2064                 for (VrfTables vrfTable : stale_vrfTables) {
2065                     Map<String, String> stale_fib_ent_map = new HashMap<>();
2066                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2067                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2068                             //Stale marking and cleanup is only meant for the routes learned through BGP.
2069                             continue;
2070                         }
2071                         if (Thread.interrupted()) {
2072                             break;
2073                         }
2074                         totalStaledCount++;
2075                         //Create MAP from stale_vrfTables.
2076                         for (String nextHop : vrfEntry.getNextHopAddressList()) {
2077                             stale_fib_ent_map.put(vrfEntry.getDestPrefix(), nextHop + "/" + vrfEntry.getLabel());
2078                         }
2079                     }
2080                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), stale_fib_ent_map);
2081                 }
2082             } else {
2083                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
2084             }
2085         } catch (Exception e) {
2086             LOG.error("createStaleFibMap:: error ", e);
2087         }
2088         LOG.error("created {} staled entries ", totalStaledCount);
2089     }
2090
2091     //map<rd, map<prefix/len, nexthop/label>>
2092     public static Map<String, Map<String, String>> getStaledFibEntriesMap() {
2093         return staledFibEntriesMap;
2094     }
2095
2096     //TODO: below function is for testing purpose with cli
2097     public static void onUpdateWithdrawRoute(String rd, String prefix, int plen) {
2098         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
2099         try {
2100             fibDSWriter.removeFibEntryFromDS(rd, prefix + "/" + plen);
2101         } catch (Throwable e) {
2102             LOG.error("failed to handle withdraw route ", e);
2103         }
2104     }
2105     public boolean isBgpConnected(){
2106         return bgpRouter.isBgpConnected();
2107     }
2108     public long getLastConnectedTS() {
2109         return bgpRouter.getLastConnectedTS();
2110     }
2111     public long getConnectTS() {
2112         return bgpRouter.getConnectTS();
2113     }
2114     public long getStartTS() {
2115         return bgpRouter.getStartTS();
2116     }
2117     public static int getTotalStaledCount() {return totalStaledCount;}
2118     public static int getTotalCleared() { return totalCleared;}
2119
2120     public Timer getBgpCountersTimer() {
2121         return bgpCountersTimer;
2122     }
2123
2124     public BgpCounters getBgpCounters() {
2125         return bgpCounters;
2126     }
2127
2128     public  void setBgpCountersTimer (Timer t) {
2129         bgpCountersTimer = t;
2130     }
2131     public  void setBgpAlarmsTimer (Timer t) {
2132         bgpAlarmsTimer = t;
2133     }
2134
2135     public void startBgpCountersTask() {
2136         if (getBgpCounters() == null) {
2137
2138             try {
2139                 bgpCounters = new BgpCounters(bgpConfigurationManager.getBgpSdncMipIp());
2140                 setBgpCountersTimer(new Timer(true));
2141                 getBgpCountersTimer().scheduleAtFixedRate(bgpCounters, 0, 120 * 1000);
2142
2143
2144                 LOG.info("Bgp Counters task scheduled for every two minutes.");
2145             } catch (Exception e) {
2146                 System.out.println("Could not start the timertask for Bgp Counters.");
2147                 e.printStackTrace();
2148             }
2149
2150             try {
2151                 bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
2152             } catch (Exception e) {
2153                 System.out.println("Could not set the default options for logging");
2154             }
2155         }
2156     }
2157
2158     public void stopBgpCountersTask() {
2159         Timer t = getBgpCountersTimer();
2160         if (getBgpCounters() != null) {
2161             t.cancel();
2162             setBgpCountersTimer(null);
2163             bgpCounters = null;
2164         }
2165     }
2166     public void startBgpAlarmsTask() {
2167         if (getBgpAlarms() == null) {
2168             try {
2169                 bgpAlarms = new BgpAlarms(this);
2170                 setBgpAlarmsTimer(new Timer(true));
2171                 getBgpAlarmsTimer().scheduleAtFixedRate(bgpAlarms, 0, 60 * 1000);
2172                 LOG.info("Bgp Alarms task scheduled for every minute.");
2173             } catch (Exception e) {
2174                 System.out.println("Could not start the timertask for Bgp Alarms.");
2175                 e.printStackTrace();
2176             }
2177         }
2178     }
2179
2180     public void stopBgpAlarmsTask() {
2181         Timer t = getBgpAlarmsTimer();
2182         if (getBgpAlarms() != null) {
2183             t.cancel();
2184             setBgpAlarmsTimer(null);
2185             bgpAlarms = null;
2186         }
2187     }
2188     public Timer getBgpAlarmsTimer() {
2189         return bgpAlarmsTimer;
2190     }
2191
2192     public BgpAlarms getBgpAlarms() {
2193         return bgpAlarms;
2194     }
2195
2196 }