vpnservice/bgpmanager/bgpmanager-impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright (c) 2015 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import com.google.common.base.Optional;
  11 import io.netty.util.concurrent.GlobalEventExecutor;
  12 import java.io.BufferedReader;
  13 import java.io.File;
  14 import java.io.FileInputStream;
  15 import java.io.IOException;
  16 import java.io.InputStreamReader;
  17 import java.lang.reflect.Constructor;
  18 import java.net.InetAddress;
  19 import java.net.NetworkInterface;
  20 import java.util.Arrays;
  21 import java.util.Enumeration;
  22 import java.util.HashMap;
  23 import java.util.Iterator;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Timer;
  27 import java.util.TimerTask;
  28 import java.util.concurrent.Callable;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.CountDownLatch;
  31 import java.util.concurrent.ExecutionException;
  32 import java.util.concurrent.Executors;
  33 import java.util.concurrent.Future;
  34 import java.util.concurrent.ScheduledExecutorService;
  35 import java.util.concurrent.TimeoutException;
  36 import java.util.concurrent.atomic.AtomicBoolean;
  37 import org.opendaylight.controller.config.api.osgi.WaitingServiceTracker;
  38 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  41 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipChange;
  42 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipListener;
  43 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipService;
  44 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  45 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  46 import org.opendaylight.genius.utils.clustering.EntityOwnerUtils;
  47 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  48 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  49 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  50 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  51 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  52 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  53 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  54 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  55 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  56 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  57 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  58 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  59 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  60 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  61 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  62 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  63 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  64 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
  65 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
  67 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
  69 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
  71 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Neighbors;
  73 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsKey;
  75 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Networks;
  76 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksKey;
  78 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Vrfs;
  79 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsBuilder;
  80 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsKey;
  81 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamilies;
  82 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesKey;
  84 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihop;
  85 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihopBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSource;
  87 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSourceBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  89 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  93 import org.opendaylight.yangtools.concepts.ListenerRegistration;
  94 import org.opendaylight.yangtools.yang.binding.DataObject;
  95 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  96 import org.opendaylight.genius.utils.batching.DefaultBatchHandler;
  97 import org.osgi.framework.BundleContext;
  98 import org.slf4j.Logger;
  99 import org.slf4j.LoggerFactory;
 100 import java.util.concurrent.atomic.AtomicInteger;
 101 import org.opendaylight.yangtools.yang.binding.DataObject;
 102
 103 public class BgpConfigurationManager {
 104     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 105     private static DataBroker dataBroker;
 106     private static FibDSWriter fibDSWriter;
 107     public static IBgpManager bgpManager;
 108     private final BundleContext bundleContext;
 109     private static Bgp config;
 110     private static BgpRouter bgpRouter;
 111     private static BgpThriftService updateServer;
 112     private BgpCounters bgpCounters;
 113     private BgpAlarms bgpAlarms;
 114     private Timer bgpCountersTimer;
 115     private Timer bgpAlarmsTimer;
 116     private static final String DEF_LOGFILE = "/var/log/bgp_debug.log";
 117     private static final String DEF_LOGLEVEL = "errors";
 118     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 119     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 120     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 121     private static final String DEF_UPORT = "6644";
 122     private static final String DEF_CHOST = "127.0.0.1";
 123     private static final String DEF_CPORT = "7644";
 124     private static final String SDNC_BGP_MIP = "sdnc_bgp_mip";
 125     private static final String BGP_SDNC_MIP = "bgp_sdnc_mip";
 126     private static final String CLUSTER_CONF_FILE = "/cluster/etc/cluster.conf";
 127     private static final Timer ipActivationCheckTimer = new Timer();
 128     private static final int STALE_FIB_WAIT = 60;
 129     private static final int RESTART_DEFAULT_GR = 90;
 130     private long StaleStartTime = 0;
 131     private long StaleEndTime = 0;
 132     private long CfgReplayStartTime = 0;
 133     private long CfgReplayEndTime = 0;
 134     private long StaleCleanupTime = 0;
 135     private static final int dsRetryCoount = 100; //100 retries, each after waitTimeBetweenEachTryMillis seconds
 136     private static final long waitTimeBetweenEachTryMillis = 1000L; //one second sleep after every retry
 137
 138     public String getBgpSdncMipIp() { return readThriftIpForCommunication(BGP_SDNC_MIP);}
 139     public long getStaleCleanupTime() {
 140         return StaleCleanupTime;
 141     }
 142
 143     public void setStaleCleanupTime(long staleCleanupTime) {
 144         StaleCleanupTime = staleCleanupTime;
 145     }
 146
 147     public long getCfgReplayEndTime() {
 148         return CfgReplayEndTime;
 149     }
 150
 151     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 152         CfgReplayEndTime = cfgReplayEndTime;
 153     }
 154
 155     public long getCfgReplayStartTime() {
 156         return CfgReplayStartTime;
 157     }
 158
 159     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 160         CfgReplayStartTime = cfgReplayStartTime;
 161     }
 162
 163     public long getStaleEndTime() {
 164         return StaleEndTime;
 165     }
 166
 167     public void setStaleEndTime(long staleEndTime) {
 168         StaleEndTime = staleEndTime;
 169     }
 170
 171     public long getStaleStartTime() {
 172         return StaleStartTime;
 173     }
 174
 175     public void setStaleStartTime(long staleStartTime) {
 176         StaleStartTime = staleStartTime;
 177     }
 178
 179
 180     // to have stale FIB map (RD, Prefix)
 181     //  number of seconds wait for route sync-up between ODL and BGP.
 182     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 183
 184     static String odlThriftIp = "127.0.0.1";
 185     static String bgpThriftIp = "127.0.0.1";
 186     private static String cHostStartup;
 187     private static String cPortStartup;
 188     private static CountDownLatch initer = new CountDownLatch(1);
 189     //static IITMProvider itmProvider;
 190     //map<rd, map<prefix/len, nexthop/label>>
 191     private static Map<String, Map<String, String>> staledFibEntriesMap = new ConcurrentHashMap<>();
 192
 193     static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 194     static final String BGP_ENTITY_NAME = "bgp";
 195
 196     static int totalStaledCount = 0;
 197     static int totalCleared = 0;
 198
 199     private static final Class[] reactors = {
 200             ConfigServerReactor.class, AsIdReactor.class,
 201             GracefulRestartReactor.class, LoggingReactor.class,
 202             NeighborsReactor.class, UpdateSourceReactor.class,
 203             EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 204             NetworksReactor.class, VrfsReactor.class, BgpReactor.class
 205     };
 206
 207     private ListenerRegistration<DataChangeListener>[] registrations;
 208
 209     final BgpConfigurationManager bgpConfigurationManager;
 210
 211     public BgpConfigurationManager(final DataBroker dataBroker,
 212                                    final EntityOwnershipService entityOwnershipService,
 213                                    final FibDSWriter fibDSWriter,
 214                                    final BundleContext bundleContext)
 215             throws InterruptedException, ExecutionException, TimeoutException {
 216         BgpConfigurationManager.dataBroker = dataBroker;
 217         BgpConfigurationManager.fibDSWriter = fibDSWriter;
 218         this.bundleContext = bundleContext;
 219         String uPort = getProperty(UPDATE_PORT, DEF_UPORT);
 220         cHostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 221         cPortStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 222         LOG.info("UpdateServer at localhost:" + uPort + " ConfigServer at "
 223                 + cHostStartup + ":" + cPortStartup);
 224         VtyshCli.setHostAddr(cHostStartup);
 225         ClearBgpCli.setHostAddr(cHostStartup);
 226         setEntityOwnershipService(entityOwnershipService);
 227         bgpRouter = BgpRouter.getInstance();
 228         odlThriftIp = readThriftIpForCommunication(SDNC_BGP_MIP);
 229         bgpThriftIp = readThriftIpForCommunication(BGP_SDNC_MIP);
 230         registerCallbacks();
 231
 232         LOG.info("BGP Configuration manager initialized");
 233         initer.countDown();
 234
 235         bgpConfigurationManager = this;
 236         BgpUtil.batchSize = BgpUtil.BATCH_SIZE;
 237         if (Integer.getInteger("batch.size") != null) {
 238             BgpUtil.batchSize = Integer.getInteger("batch.size");
 239         }
 240         BgpUtil.batchInterval = BgpUtil.PERIODICITY;
 241         if (Integer.getInteger("batch.wait.time") != null) {
 242             BgpUtil.batchInterval = Integer.getInteger("batch.wait.time");
 243         }
 244         BgpUtil.registerWithBatchManager(new DefaultBatchHandler(dataBroker, LogicalDatastoreType.CONFIGURATION, BgpUtil.batchSize, BgpUtil.batchInterval));
 245
 246         GlobalEventExecutor.INSTANCE.execute(new Runnable() {
 247             @Override
 248             public void run() {
 249                 final WaitingServiceTracker<IBgpManager> tracker = WaitingServiceTracker.create(
 250                         IBgpManager.class, bundleContext);
 251                 bgpManager = tracker.waitForService(WaitingServiceTracker.FIVE_MINUTES);
 252                 updateServer = new BgpThriftService(Integer.parseInt(uPort), bgpManager, fibDSWriter);
 253                 updateServer.start();
 254                 LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 255             }
 256         });
 257     }
 258
 259     private Object createListener(Class<?> cls) {
 260         Constructor<?> ctor;
 261         Object obj = null;
 262
 263         try {
 264             ctor = cls.getConstructor(BgpConfigurationManager.class);
 265             obj = ctor.newInstance(this);
 266         } catch (Exception e) {
 267             LOG.error("Failed to create listener object", e);
 268         }
 269         return obj;
 270     }
 271
 272     private void registerCallbacks() {
 273         String emsg = "Failed to register listener";
 274         registrations = new ListenerRegistration[reactors.length];
 275         InstanceIdentifier<?> iid = InstanceIdentifier.create(Bgp.class);
 276         for (Class reactor : reactors) {
 277             Object obj = createListener(reactor);
 278             String dclName = obj.getClass().getName();
 279             try {
 280                 AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 281                 dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 282             } catch (Exception e) {
 283                 LOG.error(emsg, e);
 284                 throw new IllegalStateException(emsg + " " + dclName, e);
 285             }
 286         }
 287     }
 288
 289     public void close() {
 290         if (updateServer != null) {
 291             updateServer.stop();
 292         }
 293         LOG.info("{} close", getClass().getSimpleName());
 294     }
 295
 296     private boolean configExists() throws InterruptedException, ExecutionException, TimeoutException {
 297         InstanceIdentifier.InstanceIdentifierBuilder<Bgp> iib =
 298                 InstanceIdentifier.builder(Bgp.class);
 299         InstanceIdentifier<Bgp> iid = iib.build();
 300         Optional<Bgp> b = BgpUtil.read(dataBroker,
 301                 LogicalDatastoreType.CONFIGURATION, iid);
 302         return b.isPresent();
 303     }
 304
 305     private String getProperty(String var, String def) {
 306         String s = bundleContext.getProperty(var);
 307         return (s == null ? def : s);
 308     }
 309
 310     boolean ignoreClusterDcnEventForFollower() {
 311         return !EntityOwnerUtils.amIEntityOwner(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME);
 312     }
 313
 314     public Bgp get() {
 315         config = getConfig();
 316         return config;
 317     }
 318
 319     public void setEntityOwnershipService(final EntityOwnershipService entityOwnershipService) {
 320         try {
 321             EntityOwnerUtils.registerEntityCandidateForOwnerShip(entityOwnershipService,
 322                     BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME, new EntityOwnershipListener() {
 323                 @Override
 324                 public void ownershipChanged(EntityOwnershipChange ownershipChange) {
 325                     LOG.trace("entity owner change event fired");
 326                     if (ownershipChange.hasOwner() && ownershipChange.isOwner()) {
 327                         LOG.trace("This PL is the Owner");
 328                         activateMIP();
 329                         bgpRestarted();
 330                     } else {
 331                         LOG.info("Not owner: hasOwner: {}, isOwner: {}",ownershipChange.hasOwner(),
 332                                 ownershipChange.isOwner() );
 333                     }
 334                 }
 335             });
 336         } catch (Exception e) {
 337             LOG.error("failed to register bgp entity", e);
 338         }
 339     }
 340
 341     private static final String addWarn =
 342             "Config store updated; undo with Delete if needed.";
 343     private static final String delWarn =
 344             "Config store updated; undo with Add if needed.";
 345     private static final String updWarn =
 346             "Update operation not supported; Config store updated;"
 347                     + " restore with another Update if needed.";
 348
 349     public class ConfigServerReactor
 350             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 351             implements AutoCloseable, ClusteredDataTreeChangeListener <ConfigServer> {
 352         private static final String yangObj = "config-server ";
 353
 354         public ConfigServerReactor() {
 355             super(ConfigServer.class, ConfigServerReactor.class);
 356         }
 357
 358         @Override
 359         protected synchronized void
 360         add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 361             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 362             if (ignoreClusterDcnEventForFollower()) {
 363                 return;
 364             }
 365
 366             try {
 367                 initer.await();
 368             } catch (Exception e) {
 369             }
 370             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 371             synchronized (BgpConfigurationManager.this) {
 372                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 373                         val.getPort().intValue());
 374                 if (!res) {
 375                     LOG.error(yangObj + "Add failed; " + addWarn);
 376                 }
 377             }
 378         }
 379
 380         @Override
 381         protected ConfigServerReactor getDataTreeChangeListener() {
 382             return ConfigServerReactor.this;
 383         }
 384
 385         @Override
 386         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 387             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 388         }
 389
 390         @Override
 391         protected synchronized void
 392         remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 393             LOG.trace("received bgp disconnect");
 394             if (ignoreClusterDcnEventForFollower()) {
 395                 return;
 396             }
 397             synchronized (BgpConfigurationManager.this) {
 398                 bgpRouter.disconnect();
 399             }
 400         }
 401
 402         @Override
 403         protected void update(InstanceIdentifier<ConfigServer> iid,
 404                               ConfigServer oldval, ConfigServer newval) {
 405             LOG.trace("received bgp Connection update");
 406             if (ignoreClusterDcnEventForFollower()) {
 407                 return;
 408             }
 409             LOG.error(yangObj + updWarn);
 410         }
 411
 412         @Override
 413         public void close() {
 414             try {
 415                 super.close();
 416             } catch (Exception e) {
 417                 LOG.error("ConfigServerReactor failed to close: ", e);
 418             }
 419         }
 420     }
 421
 422     private BgpRouter getClient(String yangObj) {
 423         if (bgpRouter == null) {
 424             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 425         }
 426         return bgpRouter;
 427     }
 428
 429     public class AsIdReactor
 430             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 431             implements AutoCloseable, ClusteredDataTreeChangeListener<AsId> {
 432
 433         private static final String yangObj = "as-id ";
 434
 435         public AsIdReactor() {
 436             super(AsId.class, AsIdReactor.class);
 437         }
 438
 439         @Override
 440         protected synchronized void
 441         add(InstanceIdentifier<AsId> iid, AsId val) {
 442             LOG.error("received bgp add asid {}",val);
 443             if (ignoreClusterDcnEventForFollower()) {
 444                 return;
 445             }
 446             LOG.debug("received add router config asNum {}", val.getLocalAs().longValue());
 447             synchronized (BgpConfigurationManager.this) {
 448                 BgpRouter br = getClient(yangObj);
 449                 if (br == null) {
 450                     LOG.error("no bgp router client found exiting asid add");
 451                     return;
 452                 }
 453                 long asNum = val.getLocalAs().longValue();
 454                 IpAddress routerId = val.getRouterId();
 455                 Boolean afb = val.isAnnounceFbit();
 456                 String rid = (routerId == null) ? "" : new String(routerId.getValue());
 457                 int stalepathTime = (int) getStalePathtime(RESTART_DEFAULT_GR, val);
 458                 boolean announceFbit = true;
 459                 try {
 460                     br.startBgp(asNum, rid, stalepathTime, announceFbit);
 461                     if (getBgpCounters() == null) {
 462                         startBgpCountersTask();
 463                     }
 464                     if (getBgpAlarms() == null) {
 465                         startBgpAlarmsTask();
 466                     }
 467                 } catch (BgpRouterException bre) {
 468                     if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
 469                         LOG.error(yangObj + "Add requested when BGP is already active");
 470                     } else {
 471                         LOG.error(yangObj + "Add received exception: \""
 472                                 + bre + "\"; " + addWarn);
 473                     }
 474                 } catch (Exception e) {
 475                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; " + addWarn);
 476                 }
 477             }
 478         }
 479
 480         @Override
 481         protected AsIdReactor getDataTreeChangeListener() {
 482             return AsIdReactor.this;
 483         }
 484
 485         @Override
 486         protected InstanceIdentifier<AsId> getWildCardPath() {
 487             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 488         }
 489
 490         @Override
 491         protected synchronized void
 492         remove(InstanceIdentifier<AsId> iid, AsId val) {
 493             LOG.error("received delete router config asNum {}", val.getLocalAs().longValue());
 494             if (ignoreClusterDcnEventForFollower()) {
 495                 return;
 496             }
 497             synchronized (BgpConfigurationManager.this) {
 498                 BgpRouter br = getClient(yangObj);
 499                 if (br == null) {
 500                     return;
 501                 }
 502                 long asNum = val.getLocalAs().longValue();
 503                 try {
 504                     br.stopBgp(asNum);
 505                 } catch (Exception e) {
 506                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; " + delWarn);
 507                 }
 508                 if (getBgpCounters() != null) {
 509                     stopBgpCountersTask();
 510                 }
 511                 if (getBgpAlarms() != null) {
 512                     stopBgpAlarmsTask();
 513                 }
 514             }
 515         }
 516
 517         @Override
 518         protected void update(InstanceIdentifier<AsId> iid,
 519                               AsId oldval, AsId newval) {
 520             if (ignoreClusterDcnEventForFollower()) {
 521                 return;
 522             }
 523             LOG.error(yangObj + updWarn);
 524         }
 525
 526         @Override
 527         public void close() {
 528             try {
 529                 super.close();
 530             } catch (Exception e) {
 531                 e.printStackTrace();
 532             }
 533         }
 534     }
 535
 536     public class GracefulRestartReactor
 537             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 538             implements AutoCloseable, ClusteredDataTreeChangeListener<GracefulRestart> {
 539
 540         private static final String yangObj = "graceful-restart ";
 541
 542         public GracefulRestartReactor() {
 543             super(GracefulRestart.class, GracefulRestartReactor.class);
 544         }
 545
 546         @Override
 547         protected synchronized void
 548         add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 549             if (ignoreClusterDcnEventForFollower()) {
 550                 return;
 551             }
 552             synchronized (BgpConfigurationManager.this) {
 553                 BgpRouter br = getClient(yangObj);
 554                 if (br == null) {
 555                     return;
 556                 }
 557                 try {
 558                     br.addGracefulRestart(val.getStalepathTime().intValue());
 559                 } catch (Exception e) {
 560                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; " + addWarn);
 561                 }
 562             }
 563         }
 564
 565         @Override
 566         protected GracefulRestartReactor getDataTreeChangeListener() {
 567             return GracefulRestartReactor.this;
 568         }
 569
 570         @Override
 571         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 572             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 573         }
 574
 575         @Override
 576         protected synchronized void
 577         remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 578             if (ignoreClusterDcnEventForFollower()) {
 579                 return;
 580             }
 581             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 582             synchronized (BgpConfigurationManager.this) {
 583                 BgpRouter br = getClient(yangObj);
 584                 if (br == null) {
 585                     return;
 586                 }
 587                 try {
 588                     br.delGracefulRestart();
 589                 } catch (Exception e) {
 590                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 591                             + delWarn);
 592                 }
 593             }
 594         }
 595
 596         @Override
 597         protected void update(InstanceIdentifier<GracefulRestart> iid,
 598                               GracefulRestart oldval, GracefulRestart newval) {
 599             if (ignoreClusterDcnEventForFollower()) {
 600                 return;
 601             }
 602             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 603             synchronized (BgpConfigurationManager.this) {
 604                 BgpRouter br = getClient(yangObj);
 605                 if (br == null) {
 606                     return;
 607                 }
 608                 try {
 609                     br.addGracefulRestart(newval.getStalepathTime().intValue());
 610                 } catch (Exception e) {
 611                     LOG.error(yangObj + "update received exception: \"" + e + "\"; " + addWarn);
 612                 }
 613             }
 614         }
 615
 616         @Override
 617         public void close() {
 618             try {
 619                 super.close();
 620             } catch (Exception e) {
 621                 e.printStackTrace();
 622             }
 623         }
 624     }
 625
 626     public class LoggingReactor
 627             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 628             implements AutoCloseable, ClusteredDataTreeChangeListener<Logging> {
 629
 630         private static final String yangObj = "logging ";
 631
 632         public LoggingReactor() {
 633             super(Logging.class, LoggingReactor.class);
 634         }
 635
 636         @Override
 637         protected synchronized void
 638         add(InstanceIdentifier<Logging> iid, Logging val) {
 639             if (ignoreClusterDcnEventForFollower()) {
 640                 return;
 641             }
 642             synchronized (BgpConfigurationManager.this) {
 643                 BgpRouter br = getClient(yangObj);
 644                 if (br == null) {
 645                     return;
 646                 }
 647                 try {
 648                     br.setLogging(val.getFile(), val.getLevel());
 649                 } catch (Exception e) {
 650                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 651                             + addWarn);
 652                 }
 653             }
 654         }
 655
 656         @Override
 657         protected LoggingReactor getDataTreeChangeListener() {
 658             return LoggingReactor.this;
 659         }
 660
 661         @Override
 662         protected InstanceIdentifier<Logging> getWildCardPath() {
 663             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 664         }
 665
 666         @Override
 667         protected synchronized void
 668         remove(InstanceIdentifier<Logging> iid, Logging val) {
 669             if (ignoreClusterDcnEventForFollower()) {
 670                 return;
 671             }
 672             LOG.debug("received remove Logging config val {}", val.getLevel());
 673             synchronized (BgpConfigurationManager.this) {
 674                 BgpRouter br = getClient(yangObj);
 675                 if (br == null) {
 676                     return;
 677                 }
 678                 try {
 679                     br.setLogging(DEF_LOGFILE, DEF_LOGLEVEL);
 680                 } catch (Exception e) {
 681                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 682                             + delWarn);
 683                 }
 684             }
 685         }
 686
 687         @Override
 688         protected void update(InstanceIdentifier<Logging> iid,
 689                               Logging oldval, Logging newval) {
 690             if (ignoreClusterDcnEventForFollower()) {
 691                 return;
 692             }
 693             synchronized (BgpConfigurationManager.this) {
 694                 BgpRouter br = getClient(yangObj);
 695                 if (br == null) {
 696                     return;
 697                 }
 698                 try {
 699                     br.setLogging(newval.getFile(), newval.getLevel());
 700                 } catch (Exception e) {
 701                     LOG.error(yangObj + "newval received exception: \"" + e + "\"; "
 702                             + addWarn);
 703                 }
 704             }
 705         }
 706
 707         @Override
 708         public void close() {
 709             try {
 710                 super.close();
 711             } catch (Exception e) {
 712                 e.printStackTrace();
 713             }
 714         }
 715     }
 716
 717     public class NeighborsReactor
 718             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 719             implements AutoCloseable, ClusteredDataTreeChangeListener<Neighbors> {
 720
 721         private static final String yangObj = "neighbors ";
 722
 723         public NeighborsReactor() {
 724             super(Neighbors.class, NeighborsReactor.class);
 725         }
 726
 727         @Override
 728         protected synchronized void
 729         add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 730             if (ignoreClusterDcnEventForFollower()) {
 731                 return;
 732             }
 733             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 734             synchronized (BgpConfigurationManager.this) {
 735                 BgpRouter br = getClient(yangObj);
 736                 if (br == null) {
 737                     return;
 738                 }
 739                 String peerIp = val.getAddress().getValue();
 740                 long as = val.getRemoteAs().longValue();
 741                 try {
 742                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 743                     br.addNeighbor(peerIp, as);
 744
 745                 } catch (Exception e) {
 746                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 747                             + addWarn);
 748                 }
 749             }
 750         }
 751
 752         @Override
 753         protected NeighborsReactor getDataTreeChangeListener() {
 754             return NeighborsReactor.this;
 755         }
 756
 757         @Override
 758         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 759             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class);
 760         }
 761
 762         @Override
 763         protected synchronized void
 764         remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 765             if (ignoreClusterDcnEventForFollower()) {
 766                 return;
 767             }
 768             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 769             synchronized (BgpConfigurationManager.this) {
 770                 BgpRouter br = getClient(yangObj);
 771                 if (br == null) {
 772                     return;
 773                 }
 774                 String peerIp = val.getAddress().getValue();
 775                 try {
 776                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 777                     br.delNeighbor(peerIp);
 778                 } catch (Exception e) {
 779                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 780                             + delWarn);
 781                 }
 782             }
 783         }
 784
 785         @Override
 786         protected void update(InstanceIdentifier<Neighbors> iid,
 787                               Neighbors oldval, Neighbors newval) {
 788             if (ignoreClusterDcnEventForFollower()) {
 789                 return;
 790             }
 791             //purposefully nothing to do.
 792         }
 793
 794         @Override
 795         public void close() {
 796             try {
 797                 super.close();
 798             } catch (Exception e) {
 799                 e.printStackTrace();
 800             }
 801         }
 802     }
 803
 804     public class EbgpMultihopReactor
 805             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 806             implements AutoCloseable, ClusteredDataTreeChangeListener<EbgpMultihop> {
 807
 808         private static final String yangObj = "ebgp-multihop ";
 809
 810         public EbgpMultihopReactor() {
 811             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 812         }
 813
 814         @Override
 815         protected synchronized void
 816         add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 817             if (ignoreClusterDcnEventForFollower()) {
 818                 return;
 819             }
 820             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 821             synchronized (BgpConfigurationManager.this) {
 822                 BgpRouter br = getClient(yangObj);
 823                 if (br == null) {
 824                     return;
 825                 }
 826                 String peerIp = val.getPeerIp().getValue();
 827                 try {
 828                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 829                 } catch (Exception e) {
 830                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 831                             + addWarn);
 832                 }
 833             }
 834         }
 835
 836         @Override
 837         protected EbgpMultihopReactor getDataTreeChangeListener() {
 838             return EbgpMultihopReactor.this;
 839         }
 840
 841         @Override
 842         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
 843             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(EbgpMultihop.class);
 844         }
 845
 846         @Override
 847         protected synchronized void
 848         remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 849             if (ignoreClusterDcnEventForFollower()) {
 850                 return;
 851             }
 852             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 853             synchronized (BgpConfigurationManager.this) {
 854                 BgpRouter br = getClient(yangObj);
 855                 if (br == null) {
 856                     return;
 857                 }
 858                 String peerIp = val.getPeerIp().getValue();
 859                 try {
 860                     br.delEbgpMultihop(peerIp);
 861                 } catch (Exception e) {
 862                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 863                             + delWarn);
 864                 }
 865             }
 866         }
 867
 868         @Override
 869         protected void update(InstanceIdentifier<EbgpMultihop> iid,
 870                               EbgpMultihop oldval, EbgpMultihop newval) {
 871             if (ignoreClusterDcnEventForFollower()) {
 872                 return;
 873             }
 874             LOG.error(yangObj + updWarn);
 875         }
 876
 877         @Override
 878         public void close() {
 879             try {
 880                 super.close();
 881             } catch (Exception e) {
 882                 e.printStackTrace();
 883             }
 884         }
 885     }
 886
 887     public class UpdateSourceReactor
 888             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
 889             implements AutoCloseable, ClusteredDataTreeChangeListener<UpdateSource> {
 890
 891         private static final String yangObj = "update-source ";
 892
 893         public UpdateSourceReactor() {
 894             super(UpdateSource.class, UpdateSourceReactor.class);
 895         }
 896
 897         @Override
 898         protected synchronized void
 899         add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 900             if (ignoreClusterDcnEventForFollower()) {
 901                 return;
 902             }
 903             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
 904             synchronized (BgpConfigurationManager.this) {
 905                 BgpRouter br = getClient(yangObj);
 906                 if (br == null) {
 907                     return;
 908                 }
 909                 String peerIp = val.getPeerIp().getValue();
 910                 try {
 911                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
 912                 } catch (Exception e) {
 913                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 914                             + addWarn);
 915                 }
 916             }
 917         }
 918
 919         @Override
 920         protected UpdateSourceReactor getDataTreeChangeListener() {
 921             return UpdateSourceReactor.this;
 922         }
 923
 924         @Override
 925         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
 926             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(UpdateSource.class);
 927         }
 928
 929         @Override
 930         protected synchronized void
 931         remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 932             if (ignoreClusterDcnEventForFollower()) {
 933                 return;
 934             }
 935             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
 936             synchronized (BgpConfigurationManager.this) {
 937                 BgpRouter br = getClient(yangObj);
 938                 if (br == null) {
 939                     return;
 940                 }
 941                 String peerIp = val.getPeerIp().getValue();
 942                 try {
 943                     br.delUpdateSource(peerIp);
 944                 } catch (Exception e) {
 945                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 946                             + delWarn);
 947                 }
 948             }
 949         }
 950
 951         @Override
 952         protected void update(InstanceIdentifier<UpdateSource> iid,
 953                               UpdateSource oldval, UpdateSource newval) {
 954             if (ignoreClusterDcnEventForFollower()) {
 955                 return;
 956             }
 957             LOG.error(yangObj + updWarn);
 958         }
 959
 960         @Override
 961         public void close() {
 962             try {
 963                 super.close();
 964             } catch (Exception e) {
 965                 e.printStackTrace();
 966             }
 967         }
 968     }
 969
 970     public class AddressFamiliesReactor
 971             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
 972             implements AutoCloseable, ClusteredDataTreeChangeListener<AddressFamilies> {
 973
 974         private static final String yangObj = "address-families ";
 975
 976         public AddressFamiliesReactor() {
 977             super(AddressFamilies.class, AddressFamiliesReactor.class);
 978         }
 979
 980         @Override
 981         protected synchronized void
 982         add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
 983             if (ignoreClusterDcnEventForFollower()) {
 984                 return;
 985             }
 986             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
 987             synchronized (BgpConfigurationManager.this) {
 988                 BgpRouter br = getClient(yangObj);
 989                 if (br == null) {
 990                     return;
 991                 }
 992                 String peerIp = val.getPeerIp().getValue();
 993                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
 994                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
 995                 try {
 996                     br.addAddressFamily(peerIp, afi, safi);
 997                 } catch (Exception e) {
 998                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 999                             + addWarn);
1000                 }
1001             }
1002         }
1003
1004         @Override
1005         protected AddressFamiliesReactor getDataTreeChangeListener() {
1006             return AddressFamiliesReactor.this;
1007         }
1008
1009         @Override
1010         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
1011             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(AddressFamilies.class);
1012         }
1013
1014         @Override
1015         protected synchronized void
1016         remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1017             if (ignoreClusterDcnEventForFollower()) {
1018                 return;
1019             }
1020             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
1021             synchronized (BgpConfigurationManager.this) {
1022                 BgpRouter br = getClient(yangObj);
1023                 if (br == null) {
1024                     return;
1025                 }
1026                 String peerIp = val.getPeerIp().getValue();
1027                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1028                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1029                 try {
1030                     br.delAddressFamily(peerIp, afi, safi);
1031                 } catch (Exception e) {
1032                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
1033                             + delWarn);
1034                 }
1035             }
1036         }
1037
1038         @Override
1039         protected void update(InstanceIdentifier<AddressFamilies> iid,
1040                               AddressFamilies oldval, AddressFamilies newval) {
1041             if (ignoreClusterDcnEventForFollower()) {
1042                 return;
1043             }
1044             LOG.error(yangObj + updWarn);
1045         }
1046
1047         @Override
1048         public void close() {
1049             try {
1050                 super.close();
1051             } catch (Exception e) {
1052                 e.printStackTrace();
1053             }
1054         }
1055     }
1056
1057     public class NetworksReactor
1058             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
1059             implements AutoCloseable, ClusteredDataTreeChangeListener<Networks> {
1060
1061         private static final String yangObj = "networks ";
1062
1063         public NetworksReactor() {
1064             super(Networks.class, NetworksReactor.class);
1065         }
1066         @Override
1067         public NetworksReactor getDataTreeChangeListener() {
1068             return NetworksReactor.this;
1069         }
1070
1071         @Override
1072         protected synchronized void
1073         add(InstanceIdentifier<Networks> iid, Networks val) {
1074             if (ignoreClusterDcnEventForFollower()) {
1075                 return;
1076             }
1077             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1078             synchronized (BgpConfigurationManager.this) {
1079                 BgpRouter br = getClient(yangObj);
1080                 if (br == null) {
1081                     return;
1082                 }
1083                 String rd = val.getRd();
1084                 String pfxlen = val.getPrefixLen();
1085                 String nh = val.getNexthop().getValue();
1086                 Long label = val.getLabel();
1087                 int lbl = (label == null) ? qbgpConstants.LBL_NO_LABEL
1088                         : label.intValue();
1089                 try {
1090                     br.addPrefix(rd, pfxlen, nh, lbl);
1091                 } catch (Exception e) {
1092                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; " + addWarn);
1093                 }
1094             }
1095         }
1096
1097         @Override
1098         protected InstanceIdentifier<Networks> getWildCardPath() {
1099             return InstanceIdentifier.create(Bgp.class).child(Networks.class);
1100         }
1101
1102         @Override
1103         protected synchronized void
1104         remove(InstanceIdentifier<Networks> iid, Networks val) {
1105             if (ignoreClusterDcnEventForFollower()) {
1106                 return;
1107             }
1108             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1109             synchronized (BgpConfigurationManager.this) {
1110                 BgpRouter br = getClient(yangObj);
1111                 if (br == null) {
1112                     return;
1113                 }
1114                 String rd = val.getRd();
1115                 String pfxlen = val.getPrefixLen();
1116                 Long label = val.getLabel();
1117                 int lbl = (label == null) ? 0 : label.intValue();
1118                 if (rd == null && lbl > 0) {
1119                     //LU prefix is being deleted.
1120                     rd = Integer.toString(lbl);
1121                 }
1122                 try {
1123                     br.delPrefix(rd, pfxlen);
1124                 } catch (Exception e) {
1125                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
1126                             + delWarn);
1127                 }
1128             }
1129         }
1130
1131         @Override
1132         protected void update(final InstanceIdentifier<Networks> iid,
1133                               final Networks oldval, final Networks newval) {
1134             if (ignoreClusterDcnEventForFollower()) {
1135                 return;
1136             }
1137             if(oldval.equals(newval)){
1138                 //Update: OLD and New values are same, no need to trigger remove/add.
1139                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1140                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1141                 return;
1142             }
1143             LOG.debug("received update networks config val {}", newval.getPrefixLen());
1144             remove(iid, oldval);
1145             timer.schedule(new TimerTask() {
1146                 @Override
1147                 public void run() {
1148                     add(iid, newval);
1149                 }
1150             }, Integer.getInteger("bgp.nexthop.update.delay.in.secs", 5) * 1000);
1151         }
1152
1153         @Override
1154         public void close() {
1155             try {
1156                 super.close();
1157             } catch (Exception e) {
1158                 e.printStackTrace();
1159             }
1160         }
1161     }
1162
1163     static Timer timer = new Timer();
1164
1165     public class VrfsReactor
1166             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1167             implements AutoCloseable, ClusteredDataTreeChangeListener<Vrfs> {
1168
1169         private static final String yangObj = "vrfs ";
1170
1171         public VrfsReactor() {
1172             super(Vrfs.class, VrfsReactor.class);
1173         }
1174
1175         @Override
1176         protected synchronized void
1177         add(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1178             if (ignoreClusterDcnEventForFollower()) {
1179                 return;
1180             }
1181             LOG.debug("received add Vrfs config val {}", val.getRd());
1182             synchronized (BgpConfigurationManager.this) {
1183                 BgpRouter br = getClient(yangObj);
1184                 if (br == null) {
1185                     return;
1186                 }
1187                 try {
1188                     br.addVrf(val.getRd(), val.getImportRts(),
1189                             val.getExportRts());
1190                 } catch (Exception e) {
1191                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
1192                             + addWarn);
1193                 }
1194             }
1195         }
1196
1197         @Override
1198         protected VrfsReactor getDataTreeChangeListener() {
1199             return VrfsReactor.this;
1200         }
1201
1202         @Override
1203         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1204             return InstanceIdentifier.create(Bgp.class).child(Vrfs.class);
1205         }
1206
1207         @Override
1208         protected synchronized void
1209         remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1210             if (ignoreClusterDcnEventForFollower()) {
1211                 return;
1212             }
1213             LOG.debug("received remove Vrfs config val {}", val.getRd());
1214             synchronized (BgpConfigurationManager.this) {
1215                 BgpRouter br = getClient(yangObj);
1216                 if (br == null) {
1217                     return;
1218                 }
1219                 try {
1220                     br.delVrf(val.getRd());
1221                 } catch (Exception e) {
1222                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
1223                             + delWarn);
1224                 }
1225             }
1226         }
1227
1228         @Override
1229         protected void update(InstanceIdentifier<Vrfs> iid,
1230                               Vrfs oldval, Vrfs newval) {
1231             if (ignoreClusterDcnEventForFollower()) {
1232                 return;
1233             }
1234             LOG.debug("VRFS: Update getting triggered for VRFS rd {}", oldval.getRd());
1235             LOG.error(yangObj + updWarn);
1236         }
1237
1238         @Override
1239         public void close() {
1240             try {
1241                 super.close();
1242             } catch (Exception e) {
1243                 e.printStackTrace();
1244             }
1245         }
1246     }
1247
1248     Future lastCleanupJob;
1249     Future lastReplayJobFt = null;
1250     protected void activateMIP() {
1251         try {
1252             LOG.trace("BgpReactor: Executing MIP Activate command");
1253             Process process_bgp = Runtime.getRuntime().exec("cluster ip -a sdnc_bgp_mip");
1254             Process process_os = Runtime.getRuntime().exec("cluster ip -a sdnc_os_mip");
1255             LOG.trace("bgpMIP Activated");
1256
1257         } catch (IOException io) {
1258             //LOG.error("IO Exception got while activating mip:  ", io);
1259         } catch (Exception e) {
1260             //LOG.error("Exception got while activating mip: ", e);
1261         }
1262     }
1263
1264     AtomicBoolean started = new AtomicBoolean(false);
1265
1266     public class BgpReactor
1267             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1268             implements AutoCloseable, ClusteredDataTreeChangeListener<Bgp> {
1269
1270         private static final String yangObj = "Bgp ";
1271
1272         public BgpReactor() {
1273             super(Bgp.class, BgpReactor.class);
1274         }
1275
1276
1277         @Override
1278         protected synchronized void
1279         add(InstanceIdentifier<Bgp> iid, Bgp val) {
1280             LOG.error("received add Bgp config replaying the config");
1281
1282             try {
1283                 initer.await();
1284             } catch (Exception e) {
1285             }
1286             synchronized (BgpConfigurationManager.this) {
1287                 config = val;
1288                 if (ignoreClusterDcnEventForFollower()) {
1289                     return;
1290                 }
1291                 activateMIP();
1292                 if (isIpAvailable(odlThriftIp)) {
1293                     bgpRestarted();
1294                 } else {
1295                     ipActivationCheckTimer.scheduleAtFixedRate(new TimerTask() {
1296                         @Override
1297                         public void run() {
1298                             if (isIpAvailable(odlThriftIp)) {
1299                                 bgpRestarted();
1300                                 ipActivationCheckTimer.cancel();
1301                             } else {
1302                                 LOG.trace("waiting for odlThriftIP: {} to be present", odlThriftIp);
1303                             }
1304                         }
1305                     }, 10000L, 10000L);
1306                 }
1307             }
1308         }
1309
1310         @Override
1311         protected BgpReactor getDataTreeChangeListener() {
1312             return BgpReactor.this;
1313         }
1314
1315         @Override
1316         protected InstanceIdentifier<Bgp> getWildCardPath() {
1317             return InstanceIdentifier.create(Bgp.class);
1318         }
1319
1320         @Override
1321         protected synchronized void
1322         remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1323             if (ignoreClusterDcnEventForFollower()) {
1324                 return;
1325             }
1326             LOG.debug("received remove Bgp config");
1327             synchronized (BgpConfigurationManager.this) {
1328                 config = null;
1329             }
1330         }
1331
1332         @Override
1333         protected void update(InstanceIdentifier<Bgp> iid,
1334                               Bgp oldval, Bgp newval) {
1335             if (ignoreClusterDcnEventForFollower()) {
1336                 return;
1337             }
1338             synchronized (BgpConfigurationManager.this) {
1339                 config = newval;
1340             }
1341         }
1342
1343         @Override
1344         public void close() {
1345             try {
1346                 super.close();
1347             } catch (Exception e) {
1348                 e.printStackTrace();
1349             }
1350         }
1351     }
1352
1353     public String readThriftIpForCommunication( String mipAddr) {
1354         File f = new File(CLUSTER_CONF_FILE);
1355         if (!f.exists()) {
1356             return  DEF_CHOST;
1357         }
1358         BufferedReader br = null;
1359         try {
1360             br = new BufferedReader(new InputStreamReader(
1361                     new FileInputStream(f)));
1362             String line = br.readLine();
1363             while (line != null) {
1364                 if (line.contains(mipAddr)) {
1365                     line = line.trim();
1366                     return line.substring(line.lastIndexOf(" ") + 1);
1367                 }
1368                 line = br.readLine();
1369             }
1370         } catch (Exception e) {
1371         } finally {
1372             try {
1373                 br.close();
1374             } catch (Exception ignore) {
1375             }
1376         }
1377         return DEF_CHOST;
1378     }
1379
1380     public boolean isIpAvailable(String odlip) {
1381
1382         try {
1383             if (odlip != null) {
1384                 if ("127.0.0.1".equals(odlip)) {
1385                     return true;
1386                 }
1387                 Enumeration e = NetworkInterface.getNetworkInterfaces();
1388                 while (e.hasMoreElements()) {
1389                     NetworkInterface n = (NetworkInterface) e.nextElement();
1390                     Enumeration ee = n.getInetAddresses();
1391                     while (ee.hasMoreElements()) {
1392                         InetAddress i = (InetAddress) ee.nextElement();
1393                         if (odlip.equals(i.getHostAddress())) {
1394                             return true;
1395                         }
1396                         ;
1397                     }
1398                 }
1399             }
1400         } catch (Exception e) {
1401         }
1402         return false;
1403     }
1404
1405     public static long getStalePathtime(int defValue, AsId as_num) {
1406         long spt = 0;
1407         try {
1408             spt = getConfig().getGracefulRestart().getStalepathTime();
1409         } catch (Exception e) {
1410             try {
1411                 spt = as_num.getStalepathTime();
1412                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1413             } catch (Exception ignore) {
1414                 LOG.trace("BGP AS id is not set using graceful");
1415                 spt = defValue;
1416             }
1417         }
1418         if (spt == 0) {
1419             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1420             spt = defValue;
1421         }
1422         return spt;
1423     }
1424
1425     public synchronized void bgpRestarted() {
1426         /*
1427          * If there a thread which in the process of stale cleanup, cancel it
1428          * and start a new thread (to avoid processing same again).
1429          */
1430         if (previousReplayJobInProgress()) {
1431             cancelPreviousReplayJob();
1432         }
1433         Runnable task = new Runnable() {
1434             @Override
1435             public void run() {
1436                 try {
1437                     LOG.info("running bgp replay task ");
1438                     if (get() == null) {
1439                         String host = getConfigHost();
1440                         int port = getConfigPort();
1441                         LOG.info("connecting  to bgp host {} ", host);
1442
1443                         boolean res = bgpRouter.connect(host, port);
1444                         LOG.info("no config to push in bgp replay task ");
1445                         return;
1446                     }
1447                     setStaleStartTime(System.currentTimeMillis());
1448                     LOG.info("started creating stale fibDSWriter  map ");
1449                     createStaleFibMap();
1450                     setStaleEndTime(System.currentTimeMillis());
1451                     LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime()- getStaleStartTime());
1452                     LOG.info("started bgp config replay ");
1453                     setCfgReplayStartTime(System.currentTimeMillis());
1454                     replay();
1455                     setCfgReplayEndTime(System.currentTimeMillis());
1456                     LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1457                     long route_sync_time = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1458                     Thread.sleep(route_sync_time * 1000L);
1459                     setStaleCleanupTime(route_sync_time);
1460                     new RouteCleanup().call();
1461                 } catch (Exception eCancel) {
1462                     LOG.error("Stale Cleanup Task Cancelled", eCancel);
1463                 }
1464             }
1465         };
1466         lastReplayJobFt = executor.submit(task);
1467     }
1468
1469     private boolean previousReplayJobInProgress() {
1470         return lastReplayJobFt != null && !lastReplayJobFt.isDone();
1471     }
1472
1473     private void cancelPreviousReplayJob() {
1474         try {
1475             LOG.error("cancelling already running bgp replay task");
1476             lastReplayJobFt.cancel(true);
1477             lastReplayJobFt = null;
1478             Thread.sleep(2000);
1479         } catch (Throwable e) {
1480             LOG.error("Failed to cancel previous replay job ",e);
1481         }
1482     }
1483
1484     private static void doRouteSync() {
1485         BgpSyncHandle bsh = BgpSyncHandle.getInstance();
1486         LOG.error("Starting BGP route sync");
1487         try {
1488             bgpRouter.initRibSync(bsh);
1489         } catch (Exception e) {
1490             LOG.error("Route sync aborted, exception when initializing: " + e);
1491             return;
1492         }
1493         while (bsh.getState() != bsh.DONE) {
1494             Routes routes = null;
1495             try {
1496                 routes = bgpRouter.doRibSync(bsh);
1497             } catch (Exception e) {
1498                 LOG.error("Route sync aborted, exception when syncing: " + e);
1499                 return;
1500             }
1501             Iterator<Update> updates = routes.getUpdatesIterator();
1502             while (updates.hasNext()) {
1503                 Update u = updates.next();
1504                 Map<String, Map<String, String>> stale_fib_rd_map = BgpConfigurationManager.getStaledFibEntriesMap();
1505                 String rd = u.getRd();
1506                 String nexthop = u.getNexthop();
1507                 int label = u.getLabel();
1508                 String prefix = u.getPrefix();
1509                 int plen = u.getPrefixlen();
1510                 onUpdatePushRoute(rd, prefix, plen, nexthop, label);
1511             }
1512         }
1513         try {
1514             LOG.error("Ending BGP route-sync");
1515             bgpRouter.endRibSync(bsh);
1516         } catch (Exception e) {
1517         }
1518     }
1519
1520     /* onUpdatePushRoute
1521      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1522      *  - Entry compare shall include NextHop, Label.
1523      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1524      *  - If entry not found, add to FIB Config DS.
1525      *  - If entry found, but either Label/NextHop doesn't match.
1526      *      - Update FIB Config DS with modified values.
1527      *      - delete from Stale Map.
1528      */
1529     public static void onUpdatePushRoute(String rd, String prefix, int plen, String nextHop, int label) {
1530         Map<String, Map<String, String>> stale_fib_rd_map = BgpConfigurationManager.getStaledFibEntriesMap();
1531         boolean addroute = false;
1532         if (!stale_fib_rd_map.isEmpty()) {
1533             // restart Scenario, as MAP is not empty.
1534             Map<String, String> map = stale_fib_rd_map.get(rd);
1535             if (map != null) {
1536                 String nexthoplabel = map.get(prefix + "/" + plen);
1537                 if (null == nexthoplabel) {
1538                     // New Entry, which happened to be added during restart.
1539                     addroute = true;
1540                 } else {
1541                     map.remove(prefix + "/" + plen);
1542                     if (isRouteModified(nextHop, label, nexthoplabel)) {
1543                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1544                         // Existing entry, where in Nexthop/Label got modified during restart
1545                         addroute = true;
1546                     }
1547                 }
1548             }
1549         } else {
1550             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1551             addroute = true;
1552         }
1553         if (addroute) {
1554             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1555             fibDSWriter.addFibEntryToDS(rd, prefix + "/" + plen, Arrays.asList(nextHop), label, RouteOrigin.BGP);
1556             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1557         }
1558     }
1559
1560     private static boolean isRouteModified(String nexthop, int label, String nexthoplabel) {
1561         return !nexthoplabel.isEmpty() && !nexthoplabel.equals(nexthop + "/" + label);
1562     }
1563
1564     static private void replayNbrConfig(List<Neighbors> n, BgpRouter br) {
1565         for (Neighbors nbr : n) {
1566             try {
1567                 br.addNeighbor(nbr.getAddress().getValue(),
1568                         nbr.getRemoteAs().longValue());
1569                 //itmProvider.buildTunnelsToDCGW(new IpAddress(nbr.getAddress().getValue().toCharArray()));
1570             } catch (Exception e) {
1571                 LOG.error("Replay:addNbr() received exception: \"" + e + "\"");
1572                 continue;
1573             }
1574             EbgpMultihop en = nbr.getEbgpMultihop();
1575             if (en != null) {
1576                 try {
1577                     br.addEbgpMultihop(en.getPeerIp().getValue(),
1578                             en.getNhops().intValue());
1579                 } catch (Exception e) {
1580                     LOG.error("Replay:addEBgp() received exception: \"" + e + "\"");
1581                 }
1582             }
1583             UpdateSource us = nbr.getUpdateSource();
1584             if (us != null) {
1585                 try {
1586                     br.addUpdateSource(us.getPeerIp().getValue(),
1587                             us.getSourceIp().getValue());
1588                 } catch (Exception e) {
1589                     LOG.error("Replay:addUS() received exception: \"" + e + "\"");
1590                 }
1591             }
1592             List<AddressFamilies> afs = nbr.getAddressFamilies();
1593             if (afs != null) {
1594                 for (AddressFamilies af : afs) {
1595                     af_afi afi = af_afi.findByValue(af.getAfi().intValue());
1596                     af_safi safi = af_safi.findByValue(af.getSafi().intValue());
1597                     try {
1598                         br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
1599                     } catch (Exception e) {
1600                         LOG.error("Replay:addAf() received exception: \"" + e + "\"");
1601                     }
1602                 }
1603             }
1604         }
1605     }
1606
1607     public static String getConfigHost() {
1608         if (config == null) {
1609             return cHostStartup;
1610         }
1611         ConfigServer ts = config.getConfigServer();
1612         return (ts == null ? cHostStartup : ts.getHost().getValue());
1613     }
1614
1615     public static int getConfigPort() {
1616         if (config == null) {
1617             return Integer.parseInt(cPortStartup);
1618         }
1619         ConfigServer ts = config.getConfigServer();
1620         return (ts == null ? Integer.parseInt(cPortStartup) :
1621                 ts.getPort().intValue());
1622     }
1623
1624     public static Bgp getConfig() {
1625         AtomicInteger bgpDSretryCount = new AtomicInteger(dsRetryCoount);
1626         while (0  != bgpDSretryCount.decrementAndGet()) {
1627             try {
1628                 return BgpUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION,
1629                         InstanceIdentifier.create(Bgp.class)).orNull();
1630             } catch (Exception e) {
1631                 //Config DS may not be up, so sleep for 1 second and retry
1632                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
1633                 try {
1634                     Thread.sleep(waitTimeBetweenEachTryMillis);
1635                 }catch (Exception timerEx){
1636                     LOG.debug("waitTimeBetweenEachTryMillis, Timer got interrupted while waiting for" +
1637                             "config DS availability", timerEx);
1638                 }
1639             }
1640          }
1641          LOG.error("failed to get bgp config");
1642          return null;
1643      }
1644
1645     public synchronized void replay() {
1646         synchronized (bgpConfigurationManager) {
1647             String host = getConfigHost();
1648             int port = getConfigPort();
1649             LOG.error("connecting  to bgp host {} ", host);
1650
1651             boolean res = bgpRouter.connect(host, port);
1652             if (!res) {
1653                 String msg = "Cannot connect to BGP config server at " + host + ":" + port;
1654                 if (config != null) {
1655                     msg += "; Configuration Replay aborted";
1656                 }
1657                 LOG.error(msg);
1658                 return;
1659             }
1660             config = getConfig();
1661             if (config == null) {
1662                 LOG.error("bgp config is empty nothing to push to bgp");
1663                 return;
1664             }
1665             BgpRouter br = bgpRouter;
1666             AsId a = config.getAsId();
1667             if (a == null) {
1668                 return;
1669             }
1670             long asNum = a.getLocalAs().longValue();
1671             IpAddress routerId = a.getRouterId();
1672             Long spt = a.getStalepathTime();
1673             Boolean afb = a.isAnnounceFbit();
1674             String rid = (routerId == null) ? "" : new String(routerId.getValue());
1675             int stalepathTime = (int) getStalePathtime(0, config.getAsId());
1676             boolean announceFbit = true;
1677             try {
1678                 br.startBgp(asNum, rid, stalepathTime, announceFbit);
1679             } catch (BgpRouterException bre) {
1680                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
1681                     doRouteSync();
1682                 } else {
1683                     LOG.error("Replay: startBgp() received exception: \""
1684                             + bre + "\"; " + addWarn);
1685                 }
1686             } catch (Exception e) {
1687                 //not unusual. We may have restarted & BGP is already on
1688                 LOG.error("Replay:startBgp() received exception: \"" + e + "\"");
1689             }
1690
1691             if (getBgpCounters() == null) {
1692                 startBgpCountersTask();
1693             }
1694
1695             if (getBgpAlarms() == null) {
1696                 startBgpAlarmsTask();
1697             }
1698
1699             Logging l = config.getLogging();
1700             if (l != null) {
1701                 try {
1702                     br.setLogging(l.getFile(), l.getLevel());
1703                 } catch (Exception e) {
1704                     LOG.error("Replay:setLogging() received exception: \"" + e + "\"");
1705                 }
1706             }
1707
1708             GracefulRestart g = config.getGracefulRestart();
1709             if (g != null) {
1710                 try {
1711                     br.addGracefulRestart(g.getStalepathTime().intValue());
1712                 } catch (Exception e) {
1713                     LOG.error("Replay:addGr() received exception: \"" + e + "\"");
1714                 }
1715             }
1716
1717             List<Vrfs> v = config.getVrfs();
1718             if (v != null) {
1719                 for (Vrfs vrf : v) {
1720                     try {
1721                         br.addVrf(vrf.getRd(), vrf.getImportRts(),
1722                                 vrf.getExportRts());
1723                     } catch (Exception e) {
1724                         LOG.error("Replay:addVrf() received exception: \"" + e + "\"");
1725                     }
1726                 }
1727             }
1728
1729             List<Networks> ln = config.getNetworks();
1730             if (ln != null) {
1731                 for (Networks net : ln) {
1732                     String rd = net.getRd();
1733                     String pfxlen = net.getPrefixLen();
1734                     String nh = net.getNexthop().getValue();
1735                     Long label = net.getLabel();
1736                     int lbl = (label == null) ? 0 : label.intValue();
1737                     if (rd == null && lbl > 0) {
1738                         //LU prefix is being deleted.
1739                         rd = Integer.toString(lbl);
1740                     }
1741                     try {
1742                         br.addPrefix(rd, pfxlen, nh, lbl);
1743                     } catch (Exception e) {
1744                         LOG.error("Replay:addPfx() received exception: \"" + e + "\"");
1745                     }
1746                 }
1747             }
1748             List<Neighbors> n = config.getNeighbors();
1749             if (n != null) {
1750                 LOG.error("configuring existing Neighbors present for replay total neighbors {}", n.size());
1751                 replayNbrConfig(n, br);
1752             } else {
1753                 LOG.error("no Neighbors present for replay config ");
1754             }
1755         }
1756     }
1757
1758     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
1759         BgpUtil.update(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1760     }
1761
1762     private <T extends DataObject> void asyncWrite(InstanceIdentifier<T> iid, T dto) {
1763         BgpUtil.write(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1764     }
1765
1766     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
1767         BgpUtil.delete(dataBroker, LogicalDatastoreType.CONFIGURATION, iid);
1768     }
1769
1770     public synchronized void
1771     startConfig(String bgpHost, int thriftPort) {
1772         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
1773                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
1774         InstanceIdentifier<ConfigServer> iid = iib.build();
1775         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
1776         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
1777                 .setPort((long) thriftPort).build();
1778         update(iid, dto);
1779     }
1780
1781     public synchronized void
1782     startBgp(long as, String routerId, int spt, boolean fbit) {
1783         IpAddress rid = (routerId == null) ?
1784                 null : new IpAddress(routerId.toCharArray());
1785         Long staleTime = (long) spt;
1786         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
1787                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
1788         InstanceIdentifier<AsId> iid = iib.build();
1789         AsId dto = new AsIdBuilder().setLocalAs(as)
1790                 .setRouterId(rid)
1791                 .setStalepathTime(staleTime)
1792                 .setAnnounceFbit(fbit).build();
1793         update(iid, dto);
1794     }
1795
1796     public synchronized void
1797     addLogging(String fileName, String logLevel) {
1798         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
1799                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
1800         InstanceIdentifier<Logging> iid = iib.build();
1801         Logging dto = new LoggingBuilder().setFile(fileName)
1802                 .setLevel(logLevel).build();
1803         update(iid, dto);
1804     }
1805
1806     public synchronized void
1807     addGracefulRestart(int staleTime) {
1808         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
1809                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
1810         InstanceIdentifier<GracefulRestart> iid = iib.build();
1811         GracefulRestart dto = new GracefulRestartBuilder()
1812                 .setStalepathTime((long) staleTime).build();
1813         update(iid, dto);
1814     }
1815
1816     public synchronized void
1817     addNeighbor(String nbrIp, long remoteAs) {
1818         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1819         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
1820                 InstanceIdentifier.builder(Bgp.class)
1821                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
1822         InstanceIdentifier<Neighbors> iid = iib.build();
1823         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
1824                 .setRemoteAs(remoteAs).build();
1825         update(iid, dto);
1826     }
1827
1828     public synchronized void
1829     addUpdateSource(String nbrIp, String srcIp) {
1830         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1831         Ipv4Address srcAddr = new Ipv4Address(srcIp);
1832         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
1833                 InstanceIdentifier.builder(Bgp.class)
1834                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1835                         .child(UpdateSource.class);
1836         InstanceIdentifier<UpdateSource> iid = iib.build();
1837         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
1838                 .setSourceIp(srcAddr).build();
1839         update(iid, dto);
1840     }
1841
1842     public synchronized void
1843     addEbgpMultihop(String nbrIp, int nHops) {
1844         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1845         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
1846                 InstanceIdentifier.builder(Bgp.class)
1847                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1848                         .child(EbgpMultihop.class);
1849         InstanceIdentifier<EbgpMultihop> iid = iib.build();
1850         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
1851                 .setNhops((long) nHops).build();
1852         update(iid, dto);
1853     }
1854
1855     public synchronized void
1856     addAddressFamily(String nbrIp, int afi, int safi) {
1857         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1858         Long a = (long) afi;
1859         Long sa = (long) safi;
1860         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
1861                 InstanceIdentifier.builder(Bgp.class)
1862                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1863                         .child(AddressFamilies.class, new AddressFamiliesKey(a, sa));
1864         InstanceIdentifier<AddressFamilies> iid = iib.build();
1865         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
1866                 .setAfi(a).setSafi(sa).build();
1867         update(iid, dto);
1868     }
1869
1870     public synchronized void
1871     addPrefix(String rd, String pfx, List<String> nhList, int lbl) {
1872         for (String nh : nhList) {
1873             Ipv4Address nexthop = new Ipv4Address(nh);
1874             Long label = (long) lbl;
1875             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
1876                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
1877             Networks dto = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
1878                                                 .setLabel(label).build();
1879             update(iid, dto);
1880         }
1881     }
1882
1883     public synchronized void
1884     addVrf(String rd, List<String> irts, List<String> erts) {
1885         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
1886                 InstanceIdentifier.builder(Bgp.class)
1887                         .child(Vrfs.class, new VrfsKey(rd));
1888         InstanceIdentifier<Vrfs> iid = iib.build();
1889         Vrfs dto = new VrfsBuilder().setRd(rd)
1890                 .setImportRts(irts)
1891                 .setExportRts(erts).build();
1892         BgpUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1893     }
1894
1895     public synchronized void stopConfig() {
1896         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
1897                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
1898         InstanceIdentifier<ConfigServer> iid = iib.build();
1899         delete(iid);
1900     }
1901
1902     public synchronized void stopBgp() {
1903         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
1904                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
1905         InstanceIdentifier<AsId> iid = iib.build();
1906         delete(iid);
1907     }
1908
1909     public synchronized void delLogging() {
1910         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
1911                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
1912         InstanceIdentifier<Logging> iid = iib.build();
1913         delete(iid);
1914     }
1915
1916     public synchronized void delGracefulRestart() {
1917         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
1918                 InstanceIdentifier.builder(Bgp.class)
1919                         .child(GracefulRestart.class);
1920         InstanceIdentifier<GracefulRestart> iid = iib.build();
1921         delete(iid);
1922     }
1923
1924     public synchronized void delNeighbor(String nbrIp) {
1925         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1926         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
1927                 InstanceIdentifier.builder(Bgp.class)
1928                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
1929         InstanceIdentifier<Neighbors> iid = iib.build();
1930         delete(iid);
1931     }
1932
1933     public synchronized void delUpdateSource(String nbrIp) {
1934         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1935         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
1936                 InstanceIdentifier.builder(Bgp.class)
1937                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1938                         .child(UpdateSource.class);
1939         InstanceIdentifier<UpdateSource> iid = iib.build();
1940         delete(iid);
1941     }
1942
1943     public synchronized void delEbgpMultihop(String nbrIp) {
1944         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1945         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
1946                 InstanceIdentifier.builder(Bgp.class)
1947                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1948                         .child(EbgpMultihop.class);
1949         InstanceIdentifier<EbgpMultihop> iid = iib.build();
1950         delete(iid);
1951     }
1952
1953     public synchronized void
1954     delAddressFamily(String nbrIp, int afi, int safi) {
1955         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1956         Long a = (long) afi;
1957         Long sa = (long) safi;
1958         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
1959                 InstanceIdentifier.builder(Bgp.class)
1960                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1961                         .child(AddressFamilies.class, new AddressFamiliesKey(a, sa));
1962         InstanceIdentifier<AddressFamilies> iid = iib.build();
1963         delete(iid);
1964     }
1965
1966     public synchronized void delPrefix(String rd, String pfx) {
1967         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
1968                 InstanceIdentifier.builder(Bgp.class)
1969                         .child(Networks.class, new NetworksKey(pfx, rd));
1970         InstanceIdentifier<Networks> iid = iib.build();
1971         delete(iid);
1972     }
1973
1974     public synchronized void delVrf(String rd) {
1975         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
1976                 InstanceIdentifier.builder(Bgp.class)
1977                         .child(Vrfs.class, new VrfsKey(rd));
1978         InstanceIdentifier<Vrfs> iid = iib.build();
1979         delete(iid);
1980     }
1981
1982     static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
1983
1984     /*
1985     * Remove Stale Marked Routes after timer expiry.
1986     */
1987     class RouteCleanup implements Callable<Integer> {
1988
1989         @Override
1990         public Integer call() {
1991             totalCleared = 0;
1992             try {
1993                 if (staledFibEntriesMap.isEmpty()) {
1994                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
1995                 } else {
1996                     for (String rd : staledFibEntriesMap.keySet()) {
1997                         if (Thread.interrupted()) {
1998                             return 0;
1999                         }
2000                         Map<String, String> map = staledFibEntriesMap.get(rd);
2001                         if (map != null) {
2002                             for (String prefix : map.keySet()) {
2003                                 if (Thread.interrupted()) {
2004                                     return 0;
2005                                 }
2006                                 try {
2007                                     totalCleared++;
2008                                     LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}" + rd.toString() + prefix);
2009                                     fibDSWriter.removeFibEntryFromDS(rd, prefix);
2010                                 } catch (Exception e) {
2011                                     LOG.error("BGP: RouteCleanup deletePrefix failed rd:{}, prefix{}" + rd.toString() + prefix);
2012                                 }
2013                             }
2014                         }
2015                     }
2016                 }
2017             } catch (Exception e) {
2018                 LOG.error("Cleanup Thread Got interrupted, Failed to cleanup stale routes ", e);
2019             } finally {
2020                 staledFibEntriesMap.clear();
2021             }
2022             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
2023             return 0;
2024         }
2025     }
2026
2027     /*
2028      * BGP restart scenario, ODL-BGP manager was/is running.
2029      * On re-sync notification, Get a copy of FIB database.
2030      */
2031     public static void createStaleFibMap() {
2032         totalStaledCount = 0;
2033         try {
2034             /*
2035             * at the time Stale FIB creation, Wait till all PENDING write transaction
2036              * to complete (or)wait for max timeout value of STALE_FIB_WAIT Seconds.
2037              */
2038             int retry = STALE_FIB_WAIT;
2039             while ((BgpUtil.getGetPendingWrTransaction() != 0) && (retry > 0)) {
2040                 Thread.sleep(1000);
2041                 retry--;
2042                 if (retry == 0) {
2043                     LOG.error("TimeOut occured {} seconds, in waiting stale fibDSWriter create", STALE_FIB_WAIT);
2044                 }
2045             }
2046             staledFibEntriesMap.clear();
2047             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2048             DataBroker db = BgpUtil.getBroker();
2049             if (db == null) {
2050                 LOG.error("Couldn't find BgpUtil dataBroker while creating createStaleFibMap");
2051                 return;
2052             }
2053
2054             Optional<FibEntries> fibEntries = BgpUtil.read(BgpUtil.getBroker(),
2055                     LogicalDatastoreType.CONFIGURATION, id);
2056             if (fibEntries.isPresent()) {
2057                 List<VrfTables> stale_vrfTables = fibEntries.get().getVrfTables();
2058                 for (VrfTables vrfTable : stale_vrfTables) {
2059                     Map<String, String> stale_fib_ent_map = new HashMap<>();
2060                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2061                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2062                             //Stale marking and cleanup is only meant for the routes learned through BGP.
2063                             continue;
2064                         }
2065                         if (Thread.interrupted()) {
2066                             break;
2067                         }
2068                         totalStaledCount++;
2069                         //Create MAP from stale_vrfTables.
2070                         for (String nextHop : vrfEntry.getNextHopAddressList()) {
2071                             stale_fib_ent_map.put(vrfEntry.getDestPrefix(), nextHop + "/" + vrfEntry.getLabel());
2072                         }
2073                     }
2074                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), stale_fib_ent_map);
2075                 }
2076             } else {
2077                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
2078             }
2079         } catch (Exception e) {
2080             LOG.error("createStaleFibMap:: error ", e);
2081         }
2082         LOG.error("created {} staled entries ", totalStaledCount);
2083     }
2084
2085     //map<rd, map<prefix/len, nexthop/label>>
2086     public static Map<String, Map<String, String>> getStaledFibEntriesMap() {
2087         return staledFibEntriesMap;
2088     }
2089
2090     //TODO: below function is for testing purpose with cli
2091     public static void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
2092         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
2093         try {
2094             fibDSWriter.removeFibEntryFromDS(rd, prefix + "/" + plen);
2095         } catch (Throwable e) {
2096             LOG.error("failed to handle withdraw route ", e);
2097         }
2098     }
2099     public boolean isBgpConnected(){
2100         return bgpRouter.isBgpConnected();
2101     }
2102     public long getLastConnectedTS() {
2103         return bgpRouter.getLastConnectedTS();
2104     }
2105     public long getConnectTS() {
2106         return bgpRouter.getConnectTS();
2107     }
2108     public long getStartTS() {
2109         return bgpRouter.getStartTS();
2110     }
2111     public static int getTotalStaledCount() {return totalStaledCount;}
2112     public static int getTotalCleared() { return totalCleared;}
2113
2114     public Timer getBgpCountersTimer() {
2115         return bgpCountersTimer;
2116     }
2117
2118     public BgpCounters getBgpCounters() {
2119         return bgpCounters;
2120     }
2121
2122     public  void setBgpCountersTimer (Timer t) {
2123         bgpCountersTimer = t;
2124     }
2125     public  void setBgpAlarmsTimer (Timer t) {
2126         bgpAlarmsTimer = t;
2127     }
2128
2129     public void startBgpCountersTask() {
2130         if (getBgpCounters() == null) {
2131
2132             try {
2133                 bgpCounters = new BgpCounters(bgpConfigurationManager.getBgpSdncMipIp());
2134                 setBgpCountersTimer(new Timer(true));
2135                 getBgpCountersTimer().scheduleAtFixedRate(bgpCounters, 0, 120 * 1000);
2136
2137
2138                 LOG.info("Bgp Counters task scheduled for every two minutes.");
2139             } catch (Exception e) {
2140                 System.out.println("Could not start the timertask for Bgp Counters.");
2141                 e.printStackTrace();
2142             }
2143
2144             try {
2145                 bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
2146             } catch (Exception e) {
2147                 System.out.println("Could not set the default options for logging");
2148             }
2149         }
2150     }
2151
2152     public void stopBgpCountersTask() {
2153         Timer t = getBgpCountersTimer();
2154         if (getBgpCounters() != null) {
2155             t.cancel();
2156             setBgpCountersTimer(null);
2157             bgpCounters = null;
2158         }
2159     }
2160     public void startBgpAlarmsTask() {
2161         if (getBgpAlarms() == null) {
2162             try {
2163                 bgpAlarms = new BgpAlarms(this);
2164                 setBgpAlarmsTimer(new Timer(true));
2165                 getBgpAlarmsTimer().scheduleAtFixedRate(bgpAlarms, 0, 60 * 1000);
2166                 LOG.info("Bgp Alarms task scheduled for every minute.");
2167             } catch (Exception e) {
2168                 System.out.println("Could not start the timertask for Bgp Alarms.");
2169                 e.printStackTrace();
2170             }
2171         }
2172     }
2173
2174     public void stopBgpAlarmsTask() {
2175         Timer t = getBgpAlarmsTimer();
2176         if (getBgpAlarms() != null) {
2177             t.cancel();
2178             setBgpAlarmsTimer(null);
2179             bgpAlarms = null;
2180         }
2181     }
2182     public Timer getBgpAlarmsTimer() {
2183         return bgpAlarmsTimer;
2184     }
2185
2186     public BgpAlarms getBgpAlarms() {
2187         return bgpAlarms;
2188     }
2189
2190 }