git.opendaylight Code Review - netvirt.git/blob

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import com.google.common.base.Optional;
  11 import io.netty.util.concurrent.GlobalEventExecutor;
  12 import java.io.BufferedReader;
  13 import java.io.File;
  14 import java.io.FileNotFoundException;
  15 import java.io.FileReader;
  16 import java.io.IOException;
  17 import java.lang.reflect.Constructor;
  18 import java.lang.reflect.InvocationTargetException;
  19 import java.net.InetAddress;
  20 import java.net.NetworkInterface;
  21 import java.net.SocketException;
  22 import java.util.Collections;
  23 import java.util.Enumeration;
  24 import java.util.HashMap;
  25 import java.util.Iterator;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Timer;
  29 import java.util.TimerTask;
  30 import java.util.concurrent.Callable;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.CountDownLatch;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.concurrent.Executors;
  35 import java.util.concurrent.Future;
  36 import java.util.concurrent.ScheduledExecutorService;
  37 import java.util.concurrent.TimeoutException;
  38 import java.util.concurrent.atomic.AtomicBoolean;
  39 import java.util.concurrent.atomic.AtomicInteger;
  40 import org.apache.thrift.TException;
  41 import org.opendaylight.controller.config.api.osgi.WaitingServiceTracker;
  42 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  43 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  44 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  45 import org.opendaylight.controller.md.sal.common.api.clustering.CandidateAlreadyRegisteredException;
  46 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipService;
  47 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  48 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  49 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  50 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  51 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  52 import org.opendaylight.genius.utils.batching.DefaultBatchHandler;
  53 import org.opendaylight.genius.utils.clustering.EntityOwnerUtils;
  54 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  55 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  56 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  57 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  58 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  59 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  60 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  61 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  62 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  63 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  64 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  65 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  66 import org.opendaylight.netvirt.bgpmanager.thrift.gen.protocol_type;
  67 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  68 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  69 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  70 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  71 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
  72 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EncapType;
  73 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.LayerType;
  74 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
  75 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
  77 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
  79 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
  80 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
  81 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Neighbors;
  83 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsBuilder;
  84 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsKey;
  85 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Networks;
  86 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksBuilder;
  87 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksKey;
  88 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Vrfs;
  89 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsBuilder;
  90 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsKey;
  91 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamilies;
  92 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesKey;
  94 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihop;
  95 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihopBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSource;
  97 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSourceBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  99 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 104 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 105 import org.opendaylight.yangtools.yang.binding.DataObject;
 106 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 107 import org.osgi.framework.BundleContext;
 108 import org.slf4j.Logger;
 109 import org.slf4j.LoggerFactory;
 110
 111 public class BgpConfigurationManager {
 112     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 113     private static DataBroker dataBroker;
 114     private static FibDSWriter fibDSWriter;
 115     public static IBgpManager bgpManager;
 116     private final BundleContext bundleContext;
 117     private static Bgp config;
 118     private static BgpRouter bgpRouter;
 119     private static BgpThriftService updateServer;
 120     private BgpCounters bgpCounters;
 121     private BgpAlarms bgpAlarms;
 122     private Timer bgpCountersTimer;
 123     private Timer bgpAlarmsTimer;
 124     private static final String DEF_LOGFILE = "/var/log/bgp_debug.log";
 125     private static final String DEF_LOGLEVEL = "errors";
 126     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 127     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 128     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 129     private static final String DEF_UPORT = "6644";
 130     private static final String DEF_CHOST = "127.0.0.1";
 131     private static final String DEF_CPORT = "7644";
 132     private static final String SDNC_BGP_MIP = "sdnc_bgp_mip";
 133     private static final String BGP_SDNC_MIP = "bgp_sdnc_mip";
 134     private static final String CLUSTER_CONF_FILE = "/cluster/etc/cluster.conf";
 135     private static final Timer IP_ACTIVATION_CHECK_TIMER = new Timer();
 136     private static final int STALE_FIB_WAIT = 60;
 137     private static final int RESTART_DEFAULT_GR = 90;
 138     private long staleStartTime = 0;
 139     private long staleEndTime = 0;
 140     private long cfgReplayStartTime = 0;
 141     private long cfgReplayEndTime = 0;
 142     private long staleCleanupTime = 0;
 143     private static final int DS_RETRY_COOUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
 144     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
 145
 146     public String getBgpSdncMipIp() {
 147         return readThriftIpForCommunication(BGP_SDNC_MIP);
 148     }
 149
 150     public long getStaleCleanupTime() {
 151         return staleCleanupTime;
 152     }
 153
 154     public void setStaleCleanupTime(long staleCleanupTime) {
 155         this.staleCleanupTime = staleCleanupTime;
 156     }
 157
 158     public long getCfgReplayEndTime() {
 159         return cfgReplayEndTime;
 160     }
 161
 162     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 163         this.cfgReplayEndTime = cfgReplayEndTime;
 164     }
 165
 166     public long getCfgReplayStartTime() {
 167         return cfgReplayStartTime;
 168     }
 169
 170     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 171         this.cfgReplayStartTime = cfgReplayStartTime;
 172     }
 173
 174     public long getStaleEndTime() {
 175         return staleEndTime;
 176     }
 177
 178     public void setStaleEndTime(long staleEndTime) {
 179         this.staleEndTime = staleEndTime;
 180     }
 181
 182     public long getStaleStartTime() {
 183         return staleStartTime;
 184     }
 185
 186     public void setStaleStartTime(long staleStartTime) {
 187         this.staleStartTime = staleStartTime;
 188     }
 189
 190
 191     // to have stale FIB map (RD, Prefix)
 192     //  number of seconds wait for route sync-up between ODL and BGP
 193     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 194
 195     static String odlThriftIp = "127.0.0.1";
 196     static String bgpThriftIp = "127.0.0.1";
 197     private static String cHostStartup;
 198     private static String cPortStartup;
 199     private static CountDownLatch initer = new CountDownLatch(1);
 200     //static IITMProvider itmProvider;
 201     //map<rd, map<prefix/len, nexthop/label>>
 202     private static Map<String, Map<String, String>> staledFibEntriesMap = new ConcurrentHashMap<>();
 203
 204     static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 205     static final String BGP_ENTITY_NAME = "bgp";
 206
 207     static int totalStaledCount = 0;
 208     static int totalCleared = 0;
 209
 210     private static final Class[] REACTORS = {
 211         ConfigServerReactor.class, AsIdReactor.class,
 212         GracefulRestartReactor.class, LoggingReactor.class,
 213         NeighborsReactor.class, UpdateSourceReactor.class,
 214         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 215         NetworksReactor.class, VrfsReactor.class, BgpReactor.class
 216     };
 217
 218     private ListenerRegistration<DataChangeListener>[] registrations;
 219
 220     final BgpConfigurationManager bgpConfigurationManager;
 221
 222     public BgpConfigurationManager(final DataBroker dataBroker,
 223             final EntityOwnershipService entityOwnershipService,
 224             final FibDSWriter fibDSWriter,
 225             final BundleContext bundleContext)
 226             throws InterruptedException, ExecutionException, TimeoutException {
 227         BgpConfigurationManager.dataBroker = dataBroker;
 228         BgpConfigurationManager.fibDSWriter = fibDSWriter;
 229         this.bundleContext = bundleContext;
 230         String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
 231         cHostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 232         cPortStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 233         LOG.info("UpdateServer at localhost:" + updatePort + " ConfigServer at "
 234                 + cHostStartup + ":" + cPortStartup);
 235         VtyshCli.setHostAddr(cHostStartup);
 236         ClearBgpCli.setHostAddr(cHostStartup);
 237         setEntityOwnershipService(entityOwnershipService);
 238         bgpRouter = BgpRouter.getInstance();
 239         odlThriftIp = readThriftIpForCommunication(SDNC_BGP_MIP);
 240         bgpThriftIp = readThriftIpForCommunication(BGP_SDNC_MIP);
 241         registerCallbacks();
 242
 243         LOG.info("BGP Configuration manager initialized");
 244         initer.countDown();
 245
 246         bgpConfigurationManager = this;
 247         BgpUtil.batchSize = BgpUtil.BATCH_SIZE;
 248         if (Integer.getInteger("batch.size") != null) {
 249             BgpUtil.batchSize = Integer.getInteger("batch.size");
 250         }
 251         BgpUtil.batchInterval = BgpUtil.PERIODICITY;
 252         if (Integer.getInteger("batch.wait.time") != null) {
 253             BgpUtil.batchInterval = Integer.getInteger("batch.wait.time");
 254         }
 255         BgpUtil.registerWithBatchManager(
 256                 new DefaultBatchHandler(dataBroker, LogicalDatastoreType.CONFIGURATION, BgpUtil.batchSize,
 257                         BgpUtil.batchInterval));
 258
 259         GlobalEventExecutor.INSTANCE.execute(() -> {
 260             final WaitingServiceTracker<IBgpManager> tracker = WaitingServiceTracker.create(
 261                     IBgpManager.class, bundleContext);
 262             bgpManager = tracker.waitForService(WaitingServiceTracker.FIVE_MINUTES);
 263             updateServer = new BgpThriftService(Integer.parseInt(updatePort), bgpManager, fibDSWriter);
 264             updateServer.start();
 265             LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 266         });
 267     }
 268
 269     private Object createListener(Class<?> cls) {
 270         Constructor<?> ctor;
 271         Object obj = null;
 272
 273         try {
 274             ctor = cls.getConstructor(BgpConfigurationManager.class);
 275             obj = ctor.newInstance(this);
 276         } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException
 277                 e) {
 278             LOG.error("Failed to create listener object", e);
 279         }
 280         return obj;
 281     }
 282
 283     private void registerCallbacks() {
 284         String emsg = "Failed to register listener";
 285         registrations = new ListenerRegistration[REACTORS.length];
 286         InstanceIdentifier<?> iid = InstanceIdentifier.create(Bgp.class);
 287         for (Class reactor : REACTORS) {
 288             Object obj = createListener(reactor);
 289             AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 290             dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 291         }
 292     }
 293
 294     public void close() {
 295         if (updateServer != null) {
 296             updateServer.stop();
 297         }
 298         LOG.info("{} close", getClass().getSimpleName());
 299     }
 300
 301     private boolean configExists() throws ReadFailedException {
 302         InstanceIdentifier.InstanceIdentifierBuilder<Bgp> iib =
 303                 InstanceIdentifier.builder(Bgp.class);
 304         InstanceIdentifier<Bgp> iid = iib.build();
 305         return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 306                 iid).isPresent();
 307     }
 308
 309     private String getProperty(String var, String def) {
 310         String property = bundleContext.getProperty(var);
 311         return (property == null ? def : property);
 312     }
 313
 314     boolean ignoreClusterDcnEventForFollower() {
 315         return !EntityOwnerUtils.amIEntityOwner(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME);
 316     }
 317
 318     public Bgp get() {
 319         config = getConfig();
 320         return config;
 321     }
 322
 323     public void setEntityOwnershipService(final EntityOwnershipService entityOwnershipService) {
 324         try {
 325             EntityOwnerUtils.registerEntityCandidateForOwnerShip(entityOwnershipService,
 326                 BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME, ownershipChange -> {
 327                     LOG.trace("entity owner change event fired");
 328                     if (ownershipChange.hasOwner() && ownershipChange.isOwner()) {
 329                         LOG.trace("This PL is the Owner");
 330                         activateMIP();
 331                         bgpRestarted();
 332                     } else {
 333                         LOG.info("Not owner: hasOwner: {}, isOwner: {}", ownershipChange.hasOwner(),
 334                                 ownershipChange.isOwner());
 335                     }
 336                 });
 337         } catch (CandidateAlreadyRegisteredException e) {
 338             LOG.error("failed to register bgp entity", e);
 339         }
 340     }
 341
 342     private static final String ADD_WARN =
 343             "Config store updated; undo with Delete if needed.";
 344     private static final String DEL_WARN =
 345             "Config store updated; undo with Add if needed.";
 346     private static final String UPD_WARN =
 347             "Update operation not supported; Config store updated;"
 348                     + " restore with another Update if needed.";
 349
 350     public class ConfigServerReactor
 351             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 352             implements AutoCloseable, ClusteredDataTreeChangeListener<ConfigServer> {
 353         private static final String YANG_OBJ = "config-server ";
 354
 355         public ConfigServerReactor() {
 356             super(ConfigServer.class, ConfigServerReactor.class);
 357         }
 358
 359         @Override
 360         protected synchronized void add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 361             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 362             if (ignoreClusterDcnEventForFollower()) {
 363                 return;
 364             }
 365
 366             try {
 367                 initer.await();
 368             } catch (InterruptedException e) {
 369                 // Ignored
 370             }
 371             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 372             synchronized (BgpConfigurationManager.this) {
 373                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 374                         val.getPort().intValue());
 375                 if (!res) {
 376                     LOG.error(YANG_OBJ + "Add failed; " + ADD_WARN);
 377                 }
 378             }
 379         }
 380
 381         @Override
 382         protected ConfigServerReactor getDataTreeChangeListener() {
 383             return ConfigServerReactor.this;
 384         }
 385
 386         @Override
 387         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 388             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 389         }
 390
 391         @Override
 392         protected synchronized void remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 393             LOG.trace("received bgp disconnect");
 394             if (ignoreClusterDcnEventForFollower()) {
 395                 return;
 396             }
 397             synchronized (BgpConfigurationManager.this) {
 398                 bgpRouter.disconnect();
 399             }
 400         }
 401
 402         @Override
 403         protected void update(InstanceIdentifier<ConfigServer> iid,
 404                 ConfigServer oldval, ConfigServer newval) {
 405             LOG.trace("received bgp Connection update");
 406             if (ignoreClusterDcnEventForFollower()) {
 407                 return;
 408             }
 409             LOG.error(YANG_OBJ + UPD_WARN);
 410         }
 411     }
 412
 413     private BgpRouter getClient(String yangObj) {
 414         if (bgpRouter == null) {
 415             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 416         }
 417         return bgpRouter;
 418     }
 419
 420     public class AsIdReactor
 421             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 422             implements AutoCloseable, ClusteredDataTreeChangeListener<AsId> {
 423
 424         private static final String YANG_OBJ = "as-id ";
 425
 426         public AsIdReactor() {
 427             super(AsId.class, AsIdReactor.class);
 428         }
 429
 430         @Override
 431         protected synchronized void add(InstanceIdentifier<AsId> iid, AsId val) {
 432             LOG.error("received bgp add asid {}", val);
 433             if (ignoreClusterDcnEventForFollower()) {
 434                 return;
 435             }
 436             LOG.debug("received add router config asNum {}", val.getLocalAs());
 437             synchronized (BgpConfigurationManager.this) {
 438                 BgpRouter br = getClient(YANG_OBJ);
 439                 if (br == null) {
 440                     LOG.error("no bgp router client found exiting asid add");
 441                     return;
 442                 }
 443                 long asNum = val.getLocalAs();
 444                 IpAddress routerId = val.getRouterId();
 445                 Boolean afb = val.isAnnounceFbit();
 446                 String rid = (routerId == null) ? "" : new String(routerId.getValue());
 447                 int stalepathTime = (int) getStalePathtime(RESTART_DEFAULT_GR, val);
 448                 boolean announceFbit = true;
 449                 try {
 450                     br.startBgp(asNum, rid, stalepathTime, announceFbit);
 451                     if (getBgpCounters() == null) {
 452                         startBgpCountersTask();
 453                     }
 454                     if (getBgpAlarms() == null) {
 455                         startBgpAlarmsTask();
 456                     }
 457                 } catch (BgpRouterException bre) {
 458                     if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
 459                         LOG.error(YANG_OBJ + "Add requested when BGP is already active");
 460                     } else {
 461                         LOG.error(YANG_OBJ + "Add received exception: \""
 462                                 + bre + "\"; " + ADD_WARN);
 463                     }
 464                 } catch (TException e) {
 465                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 466                 }
 467             }
 468         }
 469
 470         @Override
 471         protected AsIdReactor getDataTreeChangeListener() {
 472             return AsIdReactor.this;
 473         }
 474
 475         @Override
 476         protected InstanceIdentifier<AsId> getWildCardPath() {
 477             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 478         }
 479
 480         @Override
 481         protected synchronized void remove(InstanceIdentifier<AsId> iid, AsId val) {
 482             LOG.error("received delete router config asNum {}", val.getLocalAs());
 483             if (ignoreClusterDcnEventForFollower()) {
 484                 return;
 485             }
 486             synchronized (BgpConfigurationManager.this) {
 487                 BgpRouter br = getClient(YANG_OBJ);
 488                 if (br == null) {
 489                     return;
 490                 }
 491                 long asNum = val.getLocalAs();
 492                 try {
 493                     br.stopBgp(asNum);
 494                 } catch (TException | BgpRouterException e) {
 495                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 496                 }
 497                 if (getBgpCounters() != null) {
 498                     stopBgpCountersTask();
 499                 }
 500                 if (getBgpAlarms() != null) {
 501                     stopBgpAlarmsTask();
 502                 }
 503             }
 504         }
 505
 506         @Override
 507         protected void update(InstanceIdentifier<AsId> iid,
 508                 AsId oldval, AsId newval) {
 509             if (ignoreClusterDcnEventForFollower()) {
 510                 return;
 511             }
 512             LOG.error(YANG_OBJ + UPD_WARN);
 513         }
 514     }
 515
 516     public class GracefulRestartReactor
 517             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 518             implements AutoCloseable, ClusteredDataTreeChangeListener<GracefulRestart> {
 519
 520         private static final String YANG_OBJ = "graceful-restart ";
 521
 522         public GracefulRestartReactor() {
 523             super(GracefulRestart.class, GracefulRestartReactor.class);
 524         }
 525
 526         @Override
 527         protected synchronized void add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 528             if (ignoreClusterDcnEventForFollower()) {
 529                 return;
 530             }
 531             synchronized (BgpConfigurationManager.this) {
 532                 BgpRouter br = getClient(YANG_OBJ);
 533                 if (br == null) {
 534                     return;
 535                 }
 536                 try {
 537                     br.addGracefulRestart(val.getStalepathTime().intValue());
 538                 } catch (TException | BgpRouterException e) {
 539                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 540                 }
 541             }
 542         }
 543
 544         @Override
 545         protected GracefulRestartReactor getDataTreeChangeListener() {
 546             return GracefulRestartReactor.this;
 547         }
 548
 549         @Override
 550         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 551             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 552         }
 553
 554         @Override
 555         protected synchronized void remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 556             if (ignoreClusterDcnEventForFollower()) {
 557                 return;
 558             }
 559             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 560             synchronized (BgpConfigurationManager.this) {
 561                 BgpRouter br = getClient(YANG_OBJ);
 562                 if (br == null) {
 563                     return;
 564                 }
 565                 try {
 566                     br.delGracefulRestart();
 567                 } catch (TException | BgpRouterException e) {
 568                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 569                 }
 570             }
 571         }
 572
 573         @Override
 574         protected void update(InstanceIdentifier<GracefulRestart> iid,
 575                 GracefulRestart oldval, GracefulRestart newval) {
 576             if (ignoreClusterDcnEventForFollower()) {
 577                 return;
 578             }
 579             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 580             synchronized (BgpConfigurationManager.this) {
 581                 BgpRouter br = getClient(YANG_OBJ);
 582                 if (br == null) {
 583                     return;
 584                 }
 585                 try {
 586                     br.addGracefulRestart(newval.getStalepathTime().intValue());
 587                 } catch (TException | BgpRouterException e) {
 588                     LOG.error("{} update received exception; {}", YANG_OBJ, ADD_WARN, e);
 589                 }
 590             }
 591         }
 592     }
 593
 594     public class LoggingReactor
 595             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 596             implements AutoCloseable, ClusteredDataTreeChangeListener<Logging> {
 597
 598         private static final String YANG_OBJ = "logging ";
 599
 600         public LoggingReactor() {
 601             super(Logging.class, LoggingReactor.class);
 602         }
 603
 604         @Override
 605         protected synchronized void add(InstanceIdentifier<Logging> iid, Logging val) {
 606             if (ignoreClusterDcnEventForFollower()) {
 607                 return;
 608             }
 609             synchronized (BgpConfigurationManager.this) {
 610                 BgpRouter br = getClient(YANG_OBJ);
 611                 if (br == null) {
 612                     return;
 613                 }
 614                 try {
 615                     br.setLogging(val.getFile(), val.getLevel());
 616                 } catch (TException | BgpRouterException e) {
 617                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 618                 }
 619             }
 620         }
 621
 622         @Override
 623         protected LoggingReactor getDataTreeChangeListener() {
 624             return LoggingReactor.this;
 625         }
 626
 627         @Override
 628         protected InstanceIdentifier<Logging> getWildCardPath() {
 629             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 630         }
 631
 632         @Override
 633         protected synchronized void remove(InstanceIdentifier<Logging> iid, Logging val) {
 634             if (ignoreClusterDcnEventForFollower()) {
 635                 return;
 636             }
 637             LOG.debug("received remove Logging config val {}", val.getLevel());
 638             synchronized (BgpConfigurationManager.this) {
 639                 BgpRouter br = getClient(YANG_OBJ);
 640                 if (br == null) {
 641                     return;
 642                 }
 643                 try {
 644                     br.setLogging(DEF_LOGFILE, DEF_LOGLEVEL);
 645                 } catch (TException | BgpRouterException e) {
 646                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 647                 }
 648             }
 649         }
 650
 651         @Override
 652         protected void update(InstanceIdentifier<Logging> iid,
 653                 Logging oldval, Logging newval) {
 654             if (ignoreClusterDcnEventForFollower()) {
 655                 return;
 656             }
 657             synchronized (BgpConfigurationManager.this) {
 658                 BgpRouter br = getClient(YANG_OBJ);
 659                 if (br == null) {
 660                     return;
 661                 }
 662                 try {
 663                     br.setLogging(newval.getFile(), newval.getLevel());
 664                 } catch (TException | BgpRouterException e) {
 665                     LOG.error("{} newval received exception; {}", YANG_OBJ, ADD_WARN, e);
 666                 }
 667             }
 668         }
 669     }
 670
 671     public class NeighborsReactor
 672             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 673             implements AutoCloseable, ClusteredDataTreeChangeListener<Neighbors> {
 674
 675         private static final String YANG_OBJ = "neighbors ";
 676
 677         public NeighborsReactor() {
 678             super(Neighbors.class, NeighborsReactor.class);
 679         }
 680
 681         @Override
 682         protected synchronized void add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 683             if (ignoreClusterDcnEventForFollower()) {
 684                 return;
 685             }
 686             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 687             synchronized (BgpConfigurationManager.this) {
 688                 BgpRouter br = getClient(YANG_OBJ);
 689                 if (br == null) {
 690                     return;
 691                 }
 692                 String peerIp = val.getAddress().getValue();
 693                 long as = val.getRemoteAs();
 694                 try {
 695                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 696                     br.addNeighbor(peerIp, as);
 697
 698                 } catch (TException | BgpRouterException e) {
 699                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 700                 }
 701             }
 702         }
 703
 704         @Override
 705         protected NeighborsReactor getDataTreeChangeListener() {
 706             return NeighborsReactor.this;
 707         }
 708
 709         @Override
 710         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 711             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class);
 712         }
 713
 714         @Override
 715         protected synchronized void remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 716             if (ignoreClusterDcnEventForFollower()) {
 717                 return;
 718             }
 719             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 720             synchronized (BgpConfigurationManager.this) {
 721                 BgpRouter br = getClient(YANG_OBJ);
 722                 if (br == null) {
 723                     return;
 724                 }
 725                 String peerIp = val.getAddress().getValue();
 726                 try {
 727                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 728                     br.delNeighbor(peerIp);
 729                 } catch (TException | BgpRouterException e) {
 730                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 731                 }
 732             }
 733         }
 734
 735         @Override
 736         protected void update(InstanceIdentifier<Neighbors> iid,
 737                 Neighbors oldval, Neighbors newval) {
 738             if (ignoreClusterDcnEventForFollower()) {
 739                 return;
 740             }
 741             //purposefully nothing to do.
 742         }
 743     }
 744
 745     public class EbgpMultihopReactor
 746             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 747             implements AutoCloseable, ClusteredDataTreeChangeListener<EbgpMultihop> {
 748
 749         private static final String YANG_OBJ = "ebgp-multihop ";
 750
 751         public EbgpMultihopReactor() {
 752             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 753         }
 754
 755         @Override
 756         protected synchronized void add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 757             if (ignoreClusterDcnEventForFollower()) {
 758                 return;
 759             }
 760             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 761             synchronized (BgpConfigurationManager.this) {
 762                 BgpRouter br = getClient(YANG_OBJ);
 763                 if (br == null) {
 764                     return;
 765                 }
 766                 String peerIp = val.getPeerIp().getValue();
 767                 try {
 768                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 769                 } catch (TException | BgpRouterException e) {
 770                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 771                 }
 772             }
 773         }
 774
 775         @Override
 776         protected EbgpMultihopReactor getDataTreeChangeListener() {
 777             return EbgpMultihopReactor.this;
 778         }
 779
 780         @Override
 781         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
 782             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(EbgpMultihop.class);
 783         }
 784
 785         @Override
 786         protected synchronized void remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 787             if (ignoreClusterDcnEventForFollower()) {
 788                 return;
 789             }
 790             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 791             synchronized (BgpConfigurationManager.this) {
 792                 BgpRouter br = getClient(YANG_OBJ);
 793                 if (br == null) {
 794                     return;
 795                 }
 796                 String peerIp = val.getPeerIp().getValue();
 797                 try {
 798                     br.delEbgpMultihop(peerIp);
 799                 } catch (TException | BgpRouterException e) {
 800                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 801                 }
 802             }
 803         }
 804
 805         @Override
 806         protected void update(InstanceIdentifier<EbgpMultihop> iid,
 807                 EbgpMultihop oldval, EbgpMultihop newval) {
 808             if (ignoreClusterDcnEventForFollower()) {
 809                 return;
 810             }
 811             LOG.error(YANG_OBJ + UPD_WARN);
 812         }
 813     }
 814
 815     public class UpdateSourceReactor
 816             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
 817             implements AutoCloseable, ClusteredDataTreeChangeListener<UpdateSource> {
 818
 819         private static final String YANG_OBJ = "update-source ";
 820
 821         public UpdateSourceReactor() {
 822             super(UpdateSource.class, UpdateSourceReactor.class);
 823         }
 824
 825         @Override
 826         protected synchronized void add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 827             if (ignoreClusterDcnEventForFollower()) {
 828                 return;
 829             }
 830             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
 831             synchronized (BgpConfigurationManager.this) {
 832                 BgpRouter br = getClient(YANG_OBJ);
 833                 if (br == null) {
 834                     return;
 835                 }
 836                 String peerIp = val.getPeerIp().getValue();
 837                 try {
 838                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
 839                 } catch (TException | BgpRouterException e) {
 840                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 841                 }
 842             }
 843         }
 844
 845         @Override
 846         protected UpdateSourceReactor getDataTreeChangeListener() {
 847             return UpdateSourceReactor.this;
 848         }
 849
 850         @Override
 851         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
 852             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(UpdateSource.class);
 853         }
 854
 855         @Override
 856         protected synchronized void remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 857             if (ignoreClusterDcnEventForFollower()) {
 858                 return;
 859             }
 860             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
 861             synchronized (BgpConfigurationManager.this) {
 862                 BgpRouter br = getClient(YANG_OBJ);
 863                 if (br == null) {
 864                     return;
 865                 }
 866                 String peerIp = val.getPeerIp().getValue();
 867                 try {
 868                     br.delUpdateSource(peerIp);
 869                 } catch (TException | BgpRouterException e) {
 870                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 871                 }
 872             }
 873         }
 874
 875         @Override
 876         protected void update(InstanceIdentifier<UpdateSource> iid,
 877                 UpdateSource oldval, UpdateSource newval) {
 878             if (ignoreClusterDcnEventForFollower()) {
 879                 return;
 880             }
 881             LOG.error(YANG_OBJ + UPD_WARN);
 882         }
 883     }
 884
 885     public class AddressFamiliesReactor
 886             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
 887             implements AutoCloseable, ClusteredDataTreeChangeListener<AddressFamilies> {
 888
 889         private static final String YANG_OBJ = "address-families ";
 890
 891         public AddressFamiliesReactor() {
 892             super(AddressFamilies.class, AddressFamiliesReactor.class);
 893         }
 894
 895         @Override
 896         protected synchronized void add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
 897             if (ignoreClusterDcnEventForFollower()) {
 898                 return;
 899             }
 900             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
 901             synchronized (BgpConfigurationManager.this) {
 902                 BgpRouter br = getClient(YANG_OBJ);
 903                 if (br == null) {
 904                     return;
 905                 }
 906                 String peerIp = val.getPeerIp().getValue();
 907                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
 908                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
 909                 try {
 910                     br.addAddressFamily(peerIp, afi, safi);
 911                 } catch (TException | BgpRouterException e) {
 912                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 913                 }
 914             }
 915         }
 916
 917         @Override
 918         protected AddressFamiliesReactor getDataTreeChangeListener() {
 919             return AddressFamiliesReactor.this;
 920         }
 921
 922         @Override
 923         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
 924             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(AddressFamilies.class);
 925         }
 926
 927         @Override
 928         protected synchronized void remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
 929             if (ignoreClusterDcnEventForFollower()) {
 930                 return;
 931             }
 932             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
 933             synchronized (BgpConfigurationManager.this) {
 934                 BgpRouter br = getClient(YANG_OBJ);
 935                 if (br == null) {
 936                     return;
 937                 }
 938                 String peerIp = val.getPeerIp().getValue();
 939                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
 940                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
 941                 try {
 942                     br.delAddressFamily(peerIp, afi, safi);
 943                 } catch (TException | BgpRouterException e) {
 944                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 945                 }
 946             }
 947         }
 948
 949         @Override
 950         protected void update(InstanceIdentifier<AddressFamilies> iid,
 951                 AddressFamilies oldval, AddressFamilies newval) {
 952             if (ignoreClusterDcnEventForFollower()) {
 953                 return;
 954             }
 955             LOG.error(YANG_OBJ + UPD_WARN);
 956         }
 957     }
 958
 959     public class NetworksReactor
 960             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
 961             implements AutoCloseable, ClusteredDataTreeChangeListener<Networks> {
 962
 963         private static final String YANG_OBJ = "networks ";
 964
 965         public NetworksReactor() {
 966             super(Networks.class, NetworksReactor.class);
 967         }
 968
 969         @Override
 970         public NetworksReactor getDataTreeChangeListener() {
 971             return NetworksReactor.this;
 972         }
 973
 974         @Override
 975         protected synchronized void add(InstanceIdentifier<Networks> iid, Networks val) {
 976             if (ignoreClusterDcnEventForFollower()) {
 977                 return;
 978             }
 979             LOG.debug("received add Networks config val {}", val.getPrefixLen());
 980             synchronized (BgpConfigurationManager.this) {
 981                 BgpRouter br = getClient(YANG_OBJ);
 982                 if (br == null) {
 983                     return;
 984                 }
 985                 String rd = val.getRd();
 986                 String pfxlen = val.getPrefixLen();
 987                 String nh = val.getNexthop().getValue();
 988                 Long label = val.getLabel();
 989                 int lbl = (label == null) ? qbgpConstants.LBL_NO_LABEL
 990                         : label.intValue();
 991                 int l3vni = (val.getL3vni() == null) ? qbgpConstants.LBL_NO_LABEL
 992                         : val.getL3vni().intValue();
 993
 994                 BgpControlPlaneType protocolType = val.getBgpControlPlaneType();
 995                 int ethernetTag = val.getEthtag().intValue();
 996                 String esi = val.getEsi();
 997                 String macaddress = val.getMacaddress();
 998                 EncapType encapType = val.getEncapType();
 999                 String routerMac = val.getRoutermac();
1000
1001                 try {
1002                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, BgpUtil.convertToThriftProtocolType(protocolType),
1003                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1004                 } catch (TException | BgpRouterException e) {
1005                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1006                 }
1007             }
1008         }
1009
1010         @Override
1011         protected InstanceIdentifier<Networks> getWildCardPath() {
1012             return InstanceIdentifier.create(Bgp.class).child(Networks.class);
1013         }
1014
1015         @Override
1016         protected synchronized void remove(InstanceIdentifier<Networks> iid, Networks val) {
1017             if (ignoreClusterDcnEventForFollower()) {
1018                 return;
1019             }
1020             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1021             synchronized (BgpConfigurationManager.this) {
1022                 BgpRouter br = getClient(YANG_OBJ);
1023                 if (br == null) {
1024                     return;
1025                 }
1026                 String rd = val.getRd();
1027                 String pfxlen = val.getPrefixLen();
1028                 Long label = val.getLabel();
1029                 int lbl = (label == null) ? 0 : label.intValue();
1030                 if (rd == null && lbl > 0) {
1031                     //LU prefix is being deleted.
1032                     rd = Integer.toString(lbl);
1033                 }
1034                 try {
1035                     br.delPrefix(rd, pfxlen);
1036                 } catch (TException | BgpRouterException e) {
1037                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1038                 }
1039             }
1040         }
1041
1042         @Override
1043         protected void update(final InstanceIdentifier<Networks> iid,
1044                 final Networks oldval, final Networks newval) {
1045             if (ignoreClusterDcnEventForFollower()) {
1046                 return;
1047             }
1048             if (oldval.equals(newval)) {
1049                 //Update: OLD and New values are same, no need to trigger remove/add.
1050                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1051                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1052                 return;
1053             }
1054             LOG.debug("received update networks config val {}", newval.getPrefixLen());
1055             remove(iid, oldval);
1056             timer.schedule(new TimerTask() {
1057                 @Override
1058                 public void run() {
1059                     add(iid, newval);
1060                 }
1061             }, Integer.getInteger("bgp.nexthop.update.delay.in.secs", 5) * 1000);
1062         }
1063     }
1064
1065     static Timer timer = new Timer();
1066
1067     public class VrfsReactor
1068             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1069             implements AutoCloseable, ClusteredDataTreeChangeListener<Vrfs> {
1070
1071         private static final String YANG_OBJ = "vrfs ";
1072
1073         public VrfsReactor() {
1074             super(Vrfs.class, VrfsReactor.class);
1075         }
1076
1077         @Override
1078         protected synchronized void add(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1079             if (ignoreClusterDcnEventForFollower()) {
1080                 return;
1081             }
1082             LOG.debug("received add Vrfs config val {}", val.getRd());
1083             synchronized (BgpConfigurationManager.this) {
1084                 BgpRouter br = getClient(YANG_OBJ);
1085                 if (br == null) {
1086                     return;
1087                 }
1088                 try {
1089                     br.addVrf(val.getLayerType(), val.getRd(), val.getImportRts(),
1090                             val.getExportRts());
1091                 } catch (TException | BgpRouterException e) {
1092                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1093                 }
1094             }
1095         }
1096
1097         @Override
1098         protected VrfsReactor getDataTreeChangeListener() {
1099             return VrfsReactor.this;
1100         }
1101
1102         @Override
1103         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1104             return InstanceIdentifier.create(Bgp.class).child(Vrfs.class);
1105         }
1106
1107         @Override
1108         protected synchronized void remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1109             if (ignoreClusterDcnEventForFollower()) {
1110                 return;
1111             }
1112             LOG.debug("received remove Vrfs config val {}", val.getRd());
1113             synchronized (BgpConfigurationManager.this) {
1114                 BgpRouter br = getClient(YANG_OBJ);
1115                 if (br == null) {
1116                     return;
1117                 }
1118                 try {
1119                     br.delVrf(val.getRd());
1120                 } catch (TException | BgpRouterException e) {
1121                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1122                 }
1123             }
1124         }
1125
1126         @Override
1127         protected void update(InstanceIdentifier<Vrfs> iid,
1128                 Vrfs oldval, Vrfs newval) {
1129             if (ignoreClusterDcnEventForFollower()) {
1130                 return;
1131             }
1132             LOG.debug("VRFS: Update getting triggered for VRFS rd {}", oldval.getRd());
1133             LOG.error(YANG_OBJ + UPD_WARN);
1134         }
1135     }
1136
1137     Future lastCleanupJob;
1138     Future lastReplayJobFt = null;
1139
1140     protected void activateMIP() {
1141         try {
1142             LOG.trace("BgpReactor: Executing MIP Activate command");
1143             Process processBgp = Runtime.getRuntime().exec("cluster ip -a sdnc_bgp_mip");
1144             Process processOs = Runtime.getRuntime().exec("cluster ip -a sdnc_os_mip");
1145             LOG.trace("bgpMIP Activated");
1146
1147         } catch (IOException io) {
1148             LOG.error("IO Exception got while activating mip:  ", io);
1149         }
1150     }
1151
1152     AtomicBoolean started = new AtomicBoolean(false);
1153
1154     public class BgpReactor
1155             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1156             implements AutoCloseable, ClusteredDataTreeChangeListener<Bgp> {
1157
1158         private static final String YANG_OBJ = "Bgp ";
1159
1160         public BgpReactor() {
1161             super(Bgp.class, BgpReactor.class);
1162         }
1163
1164
1165         @Override
1166         protected synchronized void add(InstanceIdentifier<Bgp> iid, Bgp val) {
1167             LOG.error("received add Bgp config replaying the config");
1168
1169             try {
1170                 initer.await();
1171             } catch (InterruptedException e) {
1172                 // Ignored
1173             }
1174             synchronized (BgpConfigurationManager.this) {
1175                 config = val;
1176                 if (ignoreClusterDcnEventForFollower()) {
1177                     return;
1178                 }
1179                 activateMIP();
1180                 if (isIpAvailable(odlThriftIp)) {
1181                     bgpRestarted();
1182                 } else {
1183                     IP_ACTIVATION_CHECK_TIMER.scheduleAtFixedRate(new TimerTask() {
1184                         @Override
1185                         public void run() {
1186                             if (isIpAvailable(odlThriftIp)) {
1187                                 bgpRestarted();
1188                                 IP_ACTIVATION_CHECK_TIMER.cancel();
1189                             } else {
1190                                 LOG.trace("waiting for odlThriftIP: {} to be present", odlThriftIp);
1191                             }
1192                         }
1193                     }, 10000L, 10000L);
1194                 }
1195             }
1196         }
1197
1198         @Override
1199         protected BgpReactor getDataTreeChangeListener() {
1200             return BgpReactor.this;
1201         }
1202
1203         @Override
1204         protected InstanceIdentifier<Bgp> getWildCardPath() {
1205             return InstanceIdentifier.create(Bgp.class);
1206         }
1207
1208         @Override
1209         protected synchronized void remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1210             if (ignoreClusterDcnEventForFollower()) {
1211                 return;
1212             }
1213             LOG.debug("received remove Bgp config");
1214             synchronized (BgpConfigurationManager.this) {
1215                 config = null;
1216             }
1217         }
1218
1219         @Override
1220         protected void update(InstanceIdentifier<Bgp> iid,
1221                 Bgp oldval, Bgp newval) {
1222             if (ignoreClusterDcnEventForFollower()) {
1223                 return;
1224             }
1225             synchronized (BgpConfigurationManager.this) {
1226                 config = newval;
1227             }
1228         }
1229     }
1230
1231     public String readThriftIpForCommunication(String mipAddr) {
1232         File file = new File(CLUSTER_CONF_FILE);
1233         if (!file.exists()) {
1234             return DEF_CHOST;
1235         }
1236
1237         try (BufferedReader br = new BufferedReader(new FileReader(file))) {
1238             String line;
1239             while ((line = br.readLine()) != null) {
1240                 if (line.contains(mipAddr)) {
1241                     line = line.trim();
1242                     return line.substring(line.lastIndexOf(" ") + 1);
1243                 }
1244             }
1245         } catch (FileNotFoundException e) {
1246             return DEF_CHOST;
1247         } catch (IOException e) {
1248             LOG.error("Error reading {}", CLUSTER_CONF_FILE, e);
1249         }
1250         return DEF_CHOST;
1251     }
1252
1253     public boolean isIpAvailable(String odlip) {
1254
1255         try {
1256             if (odlip != null) {
1257                 if ("127.0.0.1".equals(odlip)) {
1258                     return true;
1259                 }
1260                 Enumeration networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces();
1261                 while (networkInterfaceEnumeration.hasMoreElements()) {
1262                     NetworkInterface networkInterface = (NetworkInterface) networkInterfaceEnumeration.nextElement();
1263                     Enumeration inetAddressEnumeration = networkInterface.getInetAddresses();
1264                     while (inetAddressEnumeration.hasMoreElements()) {
1265                         InetAddress inetAddress = (InetAddress) inetAddressEnumeration.nextElement();
1266                         if (odlip.equals(inetAddress.getHostAddress())) {
1267                             return true;
1268                         }
1269                     }
1270                 }
1271             }
1272         } catch (SocketException e) {
1273             // Ignored?
1274         }
1275         return false;
1276     }
1277
1278     public static long getStalePathtime(int defValue, AsId asId) {
1279         long spt = 0;
1280         try {
1281             spt = getConfig().getGracefulRestart().getStalepathTime();
1282         } catch (NullPointerException e) {
1283             try {
1284                 spt = asId.getStalepathTime();
1285                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1286             } catch (NullPointerException ignore) {
1287                 LOG.trace("BGP AS id is not set using graceful");
1288                 spt = defValue;
1289             }
1290         }
1291         if (spt == 0) {
1292             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1293             spt = defValue;
1294         }
1295         return spt;
1296     }
1297
1298     public synchronized void bgpRestarted() {
1299         /*
1300          * If there a thread which in the process of stale cleanup, cancel it
1301          * and start a new thread (to avoid processing same again).
1302          */
1303         if (previousReplayJobInProgress()) {
1304             cancelPreviousReplayJob();
1305         }
1306         Runnable task = () -> {
1307             try {
1308                 LOG.info("running bgp replay task ");
1309                 if (get() == null) {
1310                     String host = getConfigHost();
1311                     int port = getConfigPort();
1312                     LOG.info("connecting  to bgp host {} ", host);
1313
1314                     boolean res = bgpRouter.connect(host, port);
1315                     LOG.info("no config to push in bgp replay task ");
1316                     return;
1317                 }
1318                 setStaleStartTime(System.currentTimeMillis());
1319                 LOG.info("started creating stale fibDSWriter  map ");
1320                 createStaleFibMap();
1321                 setStaleEndTime(System.currentTimeMillis());
1322                 LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
1323                 LOG.info("started bgp config replay ");
1324                 setCfgReplayStartTime(System.currentTimeMillis());
1325                 try {
1326                     replay();
1327                 } catch (TimeoutException | ExecutionException e) {
1328                     LOG.error("Error while replaying routes. {}", e);
1329                 }
1330                 setCfgReplayEndTime(System.currentTimeMillis());
1331                 LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1332                 long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1333                 Thread.sleep(routeSyncTime * 1000L);
1334                 setStaleCleanupTime(routeSyncTime);
1335                 new RouteCleanup().call();
1336             } catch (InterruptedException eCancel) {
1337                 LOG.error("Stale Cleanup Task Cancelled", eCancel);
1338             }
1339         };
1340         lastReplayJobFt = executor.submit(task);
1341     }
1342
1343     private boolean previousReplayJobInProgress() {
1344         return lastReplayJobFt != null && !lastReplayJobFt.isDone();
1345     }
1346
1347     private void cancelPreviousReplayJob() {
1348         try {
1349             LOG.error("cancelling already running bgp replay task");
1350             lastReplayJobFt.cancel(true);
1351             lastReplayJobFt = null;
1352             Thread.sleep(2000);
1353         } catch (InterruptedException e) {
1354             LOG.error("Failed to cancel previous replay job ", e);
1355         }
1356     }
1357
1358     private static void doRouteSync() throws InterruptedException, TimeoutException, ExecutionException {
1359         BgpSyncHandle bsh = BgpSyncHandle.getInstance();
1360         LOG.error("Starting BGP route sync");
1361         try {
1362             bgpRouter.initRibSync(bsh);
1363         } catch (TException | BgpRouterException e) {
1364             LOG.error("Route sync aborted, exception when initializing", e);
1365             return;
1366         }
1367         while (bsh.getState() != bsh.DONE) {
1368             Routes routes = null;
1369             try {
1370                 routes = bgpRouter.doRibSync(bsh);
1371             } catch (TException | BgpRouterException e) {
1372                 LOG.error("Route sync aborted, exception when syncing", e);
1373                 return;
1374             }
1375             Iterator<Update> updates = routes.getUpdatesIterator();
1376             while (updates.hasNext()) {
1377                 Update update = updates.next();
1378                 Map<String, Map<String, String>> staleFibRdMap = BgpConfigurationManager.getStaledFibEntriesMap();
1379                 String rd = update.getRd();
1380                 String nexthop = update.getNexthop();
1381
1382                 // TODO: decide correct label here
1383                 int label = update.getL3label();
1384
1385                 String prefix = update.getPrefix();
1386                 int plen = update.getPrefixlen();
1387
1388
1389                 // TODO: protocol type will not be available in "update"
1390                 // use "rd" to query vrf table and obtain the protocol_type. Currently using PROTOCOL_EVPN as default.
1391                 onUpdatePushRoute(
1392                         protocol_type.PROTOCOL_EVPN,
1393                         rd,
1394                         prefix,
1395                         plen,
1396                         nexthop,
1397                         update.getEthtag(),
1398                         update.getEsi(),
1399                         update.getMacaddress(),
1400                         label,
1401                         update.getRoutermac()
1402                 );
1403             }
1404         }
1405         try {
1406             LOG.error("Ending BGP route-sync");
1407             bgpRouter.endRibSync(bsh);
1408         } catch (TException | BgpRouterException e) {
1409             // Ignored?
1410         }
1411     }
1412
1413     /* onUpdatePushRoute
1414      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1415      *  - Entry compare shall include NextHop, Label.
1416      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1417      *  - If entry not found, add to FIB Config DS.
1418      *  - If entry found, but either Label/NextHop doesn't match.
1419      *      - Update FIB Config DS with modified values.
1420      *      - delete from Stale Map.
1421      */
1422
1423     public static void onUpdatePushRoute(protocol_type protocolType,
1424                                          String rd,
1425                                          String prefix,
1426                                          int plen,
1427                                          String nextHop,
1428                                          int ethtag,
1429                                          String esi,
1430                                          String macaddress,
1431                                          int label,
1432                                          String routermac)
1433             throws InterruptedException, ExecutionException, TimeoutException {
1434         Map<String, Map<String, String>> staleFibRdMap = BgpConfigurationManager.getStaledFibEntriesMap();
1435         boolean addroute = false;
1436         long l3vni = 0L;
1437         VrfEntry.EncapType encapType = VrfEntry.EncapType.Mplsgre;
1438         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1439             encapType = VrfEntry.EncapType.Vxlan;
1440             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = BgpUtil.getVpnInstanceOpData(dataBroker, rd);
1441             if (vpnInstanceOpDataEntry != null) {
1442                 l3vni = vpnInstanceOpDataEntry.getL3vni();
1443             } else {
1444                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1445                 return;
1446             }
1447         }
1448         if (!staledFibEntriesMap.isEmpty()) {
1449             // restart Scenario, as MAP is not empty.
1450             Map<String, String> map = staledFibEntriesMap.get(rd);
1451             if (map != null) {
1452                 String nexthoplabel = map.get(prefix + "/" + plen);
1453                 if (null == nexthoplabel) {
1454                     // New Entry, which happened to be added during restart.
1455                     addroute = true;
1456                 } else {
1457                     map.remove(prefix + "/" + plen);
1458                     if (isRouteModified(nextHop, label, nexthoplabel)) {
1459                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1460                         // Existing entry, where in Nexthop/Label got modified during restart
1461                         addroute = true;
1462                     }
1463                 }
1464             }
1465         } else {
1466             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1467             addroute = true;
1468         }
1469         if (addroute) {
1470             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1471             // TODO: modify addFibEntryToDS signature
1472             fibDSWriter.addFibEntryToDS(rd, macaddress, prefix + "/" + plen, Collections.singletonList(nextHop),
1473                     encapType, label, l3vni, routermac, RouteOrigin.BGP);
1474             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1475         }
1476     }
1477
1478     private static boolean isRouteModified(String nexthop, int label, String nexthoplabel) {
1479         return !nexthoplabel.isEmpty() && !nexthoplabel.equals(nexthop + "/" + label);
1480     }
1481
1482     private static void replayNbrConfig(List<Neighbors> neighbors, BgpRouter br) {
1483         for (Neighbors nbr : neighbors) {
1484             try {
1485                 br.addNeighbor(nbr.getAddress().getValue(),
1486                         nbr.getRemoteAs());
1487                 //itmProvider.buildTunnelsToDCGW(new IpAddress(nbr.getAddress().getValue().toCharArray()));
1488             } catch (TException | BgpRouterException e) {
1489                 LOG.error("Replay:addNbr() received exception", e);
1490                 continue;
1491             }
1492             EbgpMultihop en = nbr.getEbgpMultihop();
1493             if (en != null) {
1494                 try {
1495                     br.addEbgpMultihop(en.getPeerIp().getValue(),
1496                             en.getNhops().intValue());
1497                 } catch (TException | BgpRouterException e) {
1498                     LOG.error("Replay:addEBgp() received exception", e);
1499                 }
1500             }
1501             UpdateSource us = nbr.getUpdateSource();
1502             if (us != null) {
1503                 try {
1504                     br.addUpdateSource(us.getPeerIp().getValue(),
1505                             us.getSourceIp().getValue());
1506                 } catch (TException | BgpRouterException e) {
1507                     LOG.error("Replay:addUS() received exception", e);
1508                 }
1509             }
1510             List<AddressFamilies> afs = nbr.getAddressFamilies();
1511             if (afs != null) {
1512                 for (AddressFamilies af : afs) {
1513                     af_afi afi = af_afi.findByValue(af.getAfi().intValue());
1514                     af_safi safi = af_safi.findByValue(af.getSafi().intValue());
1515                     try {
1516                         br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
1517                     } catch (TException | BgpRouterException e) {
1518                         LOG.error("Replay:addAf() received exception", e);
1519                     }
1520                 }
1521             }
1522         }
1523     }
1524
1525     public static String getConfigHost() {
1526         if (config == null) {
1527             return cHostStartup;
1528         }
1529         ConfigServer ts = config.getConfigServer();
1530         return (ts == null ? cHostStartup : ts.getHost().getValue());
1531     }
1532
1533     public static int getConfigPort() {
1534         if (config == null) {
1535             return Integer.parseInt(cPortStartup);
1536         }
1537         ConfigServer ts = config.getConfigServer();
1538         return (ts == null ? Integer.parseInt(cPortStartup) :
1539                 ts.getPort().intValue());
1540     }
1541
1542     public static Bgp getConfig() {
1543         AtomicInteger bgpDSretryCount = new AtomicInteger(DS_RETRY_COOUNT);
1544         while (0 != bgpDSretryCount.decrementAndGet()) {
1545             try {
1546                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
1547                         InstanceIdentifier.create(Bgp.class)).orNull();
1548             } catch (ReadFailedException e) {
1549                 //Config DS may not be up, so sleep for 1 second and retry
1550                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
1551                 try {
1552                     Thread.sleep(WAIT_TIME_BETWEEN_EACH_TRY_MILLIS);
1553                 } catch (InterruptedException timerEx) {
1554                     LOG.debug("WAIT_TIME_BETWEEN_EACH_TRY_MILLIS, Timer got interrupted while waiting for"
1555                             + "config DS availability", timerEx);
1556                 }
1557             }
1558         }
1559         LOG.error("failed to get bgp config");
1560         return null;
1561     }
1562
1563     @SuppressWarnings("checkstyle:IllegalCatch")
1564     public synchronized void replay() throws InterruptedException, TimeoutException, ExecutionException {
1565         synchronized (bgpConfigurationManager) {
1566             String host = getConfigHost();
1567             int port = getConfigPort();
1568             LOG.error("connecting  to bgp host {} ", host);
1569
1570             boolean res = bgpRouter.connect(host, port);
1571             if (!res) {
1572                 String msg = "Cannot connect to BGP config server at " + host + ":" + port;
1573                 if (config != null) {
1574                     msg += "; Configuration Replay aborted";
1575                 }
1576                 LOG.error(msg);
1577                 return;
1578             }
1579             config = getConfig();
1580             if (config == null) {
1581                 LOG.error("bgp config is empty nothing to push to bgp");
1582                 return;
1583             }
1584             BgpRouter br = bgpRouter;
1585             AsId asId = config.getAsId();
1586             if (asId == null) {
1587                 return;
1588             }
1589             long asNum = asId.getLocalAs();
1590             IpAddress routerId = asId.getRouterId();
1591             Long spt = asId.getStalepathTime();
1592             Boolean afb = asId.isAnnounceFbit();
1593             String rid = (routerId == null) ? "" : new String(routerId.getValue());
1594             int stalepathTime = (int) getStalePathtime(0, config.getAsId());
1595             boolean announceFbit = true;
1596             try {
1597                 br.startBgp(asNum, rid, stalepathTime, announceFbit);
1598             } catch (BgpRouterException bre) {
1599                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
1600                     doRouteSync();
1601                 } else {
1602                     LOG.error("Replay: startBgp() received exception: \""
1603                             + bre + "\"; " + ADD_WARN);
1604                 }
1605             } catch (TException e) {
1606                 //not unusual. We may have restarted & BGP is already on
1607                 LOG.error("Replay:startBgp() received exception: \"" + e + "\"");
1608             }
1609
1610             if (getBgpCounters() == null) {
1611                 startBgpCountersTask();
1612             }
1613
1614             if (getBgpAlarms() == null) {
1615                 startBgpAlarmsTask();
1616             }
1617
1618             Logging logging = config.getLogging();
1619             if (logging != null) {
1620                 try {
1621                     br.setLogging(logging.getFile(), logging.getLevel());
1622                 } catch (TException | BgpRouterException e) {
1623                     LOG.error("Replay:setLogging() received exception", e);
1624                 }
1625             }
1626
1627             GracefulRestart gracefulRestart = config.getGracefulRestart();
1628             if (gracefulRestart != null) {
1629                 try {
1630                     br.addGracefulRestart(gracefulRestart.getStalepathTime().intValue());
1631                 } catch (TException | BgpRouterException e) {
1632                     LOG.error("Replay:addGr() received exception", e);
1633                 }
1634             }
1635
1636             List<Vrfs> vrfs = config.getVrfs();
1637             if (vrfs != null) {
1638                 for (Vrfs vrf : vrfs) {
1639                     try {
1640                         br.addVrf(vrf.getLayerType(), vrf.getRd(), vrf.getImportRts(),
1641                                 vrf.getExportRts());
1642                     } catch (TException | BgpRouterException e) {
1643                         LOG.error("Replay:addVrf() received exception", e);
1644                     }
1645                 }
1646             }
1647
1648             List<Networks> ln = config.getNetworks();
1649             if (ln != null) {
1650                 for (Networks net : ln) {
1651                     String rd = net.getRd();
1652                     String pfxlen = net.getPrefixLen();
1653                     String nh = net.getNexthop().getValue();
1654                     Long label = net.getLabel();
1655                     int lbl = (label == null) ? 0 : label.intValue();
1656                     int l3vni = (net.getL3vni() == null) ? 0 : net.getL3vni().intValue();
1657                     if (rd == null && lbl > 0) {
1658                         //LU prefix is being deleted.
1659                         rd = Integer.toString(lbl);
1660                     }
1661
1662                     BgpControlPlaneType protocolType = net.getBgpControlPlaneType();
1663                     int ethernetTag = net.getEthtag().intValue();
1664                     String esi = net.getEsi();
1665                     String macaddress = net.getMacaddress();
1666                     EncapType encapType = net.getEncapType();
1667                     String routerMac = net.getRoutermac();
1668
1669                     try {
1670                         br.addPrefix(rd, pfxlen, nh, lbl, l3vni, BgpUtil.convertToThriftProtocolType(protocolType),
1671                                 ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1672                     } catch (Exception e) {
1673                         LOG.error("Replay:addPfx() received exception", e);
1674                     }
1675                 }
1676             }
1677             List<Neighbors> neighbors = config.getNeighbors();
1678             if (neighbors != null) {
1679                 LOG.error("configuring existing Neighbors present for replay total neighbors {}", neighbors.size());
1680                 replayNbrConfig(neighbors, br);
1681             } else {
1682                 LOG.error("no Neighbors present for replay config ");
1683             }
1684         }
1685     }
1686
1687     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
1688         BgpUtil.update(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1689     }
1690
1691     private <T extends DataObject> void asyncWrite(InstanceIdentifier<T> iid, T dto) {
1692         BgpUtil.write(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1693     }
1694
1695     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
1696         BgpUtil.delete(dataBroker, LogicalDatastoreType.CONFIGURATION, iid);
1697     }
1698
1699     public synchronized void startConfig(String bgpHost, int thriftPort) {
1700         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
1701                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
1702         InstanceIdentifier<ConfigServer> iid = iib.build();
1703         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
1704         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
1705                 .setPort((long) thriftPort).build();
1706         update(iid, dto);
1707     }
1708
1709     public synchronized void startBgp(long as, String routerId, int spt, boolean fbit) {
1710         IpAddress rid = (routerId == null) ? null : new IpAddress(routerId.toCharArray());
1711         Long staleTime = (long) spt;
1712         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
1713                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
1714         InstanceIdentifier<AsId> iid = iib.build();
1715         AsId dto = new AsIdBuilder().setLocalAs(as)
1716                 .setRouterId(rid)
1717                 .setStalepathTime(staleTime)
1718                 .setAnnounceFbit(fbit).build();
1719         update(iid, dto);
1720     }
1721
1722     public synchronized void addLogging(String fileName, String logLevel) {
1723         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
1724                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
1725         InstanceIdentifier<Logging> iid = iib.build();
1726         Logging dto = new LoggingBuilder().setFile(fileName)
1727                 .setLevel(logLevel).build();
1728         update(iid, dto);
1729     }
1730
1731     public synchronized void addGracefulRestart(int staleTime) {
1732         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
1733                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
1734         InstanceIdentifier<GracefulRestart> iid = iib.build();
1735         GracefulRestart dto = new GracefulRestartBuilder()
1736                 .setStalepathTime((long) staleTime).build();
1737         update(iid, dto);
1738     }
1739
1740     public synchronized void addNeighbor(String nbrIp, long remoteAs) {
1741         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1742         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
1743                 InstanceIdentifier.builder(Bgp.class)
1744                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
1745         InstanceIdentifier<Neighbors> iid = iib.build();
1746         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
1747                 .setRemoteAs(remoteAs).build();
1748         update(iid, dto);
1749     }
1750
1751     public synchronized void addUpdateSource(String nbrIp, String srcIp) {
1752         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1753         Ipv4Address srcAddr = new Ipv4Address(srcIp);
1754         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
1755                 InstanceIdentifier.builder(Bgp.class)
1756                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1757                         .child(UpdateSource.class);
1758         InstanceIdentifier<UpdateSource> iid = iib.build();
1759         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
1760                 .setSourceIp(srcAddr).build();
1761         update(iid, dto);
1762     }
1763
1764     public synchronized void addEbgpMultihop(String nbrIp, int hops) {
1765         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1766         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
1767                 InstanceIdentifier.builder(Bgp.class)
1768                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1769                         .child(EbgpMultihop.class);
1770         InstanceIdentifier<EbgpMultihop> iid = iib.build();
1771         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
1772                 .setNhops((long) hops).build();
1773         update(iid, dto);
1774     }
1775
1776     public synchronized void addAddressFamily(String nbrIp, int afi, int safi) {
1777         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1778         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
1779                 InstanceIdentifier.builder(Bgp.class)
1780                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1781                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
1782         InstanceIdentifier<AddressFamilies> iid = iib.build();
1783         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
1784                 .setAfi((long) afi).setSafi((long) safi).build();
1785         update(iid, dto);
1786     }
1787
1788     public synchronized void addPrefix(String rd, String macAddress, String pfx, List<String> nhList,
1789               VrfEntry.EncapType encapType, int lbl, long l3vni, String gatewayMac) {
1790         for (String nh : nhList) {
1791             Ipv4Address nexthop = nh != null ? new Ipv4Address(nh) : null;
1792             Long label = (long) lbl;
1793             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
1794                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
1795             NetworksBuilder networksBuilder = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
1796                                                 .setLabel(label).setEthtag(BgpConstants.DEFAULT_ETH_TAG);
1797             buildVpnEncapSpecificInfo(networksBuilder, encapType, label, l3vni, macAddress, gatewayMac);
1798             update(iid, networksBuilder.build());
1799         }
1800     }
1801
1802     private static void buildVpnEncapSpecificInfo(NetworksBuilder builder, VrfEntry.EncapType encapType, long label,
1803                                                   long l3vni, String macAddress, String gatewayMac) {
1804         if (encapType.equals(VrfEntry.EncapType.Mplsgre)) {
1805             builder.setLabel(label).setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLL3VPN)
1806                     .setEncapType(EncapType.GRE);
1807         } else {
1808             builder.setL3vni(l3vni).setMacaddress(macAddress).setRoutermac(gatewayMac)
1809                     .setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLEVPN).setEncapType(EncapType.VXLAN);
1810         }
1811     }
1812
1813     // TODO: add LayerType as arg - supports command
1814     public synchronized void addVrf(String rd, List<String> irts, List<String> erts, LayerType layerType) {
1815         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
1816                 InstanceIdentifier.builder(Bgp.class)
1817                         .child(Vrfs.class, new VrfsKey(rd));
1818         InstanceIdentifier<Vrfs> iid = iib.build();
1819         Vrfs dto = new VrfsBuilder().setRd(rd).setImportRts(irts)
1820                 .setExportRts(erts).setLayerType(layerType).build();
1821         try {
1822             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1823         } catch (TransactionCommitFailedException e) {
1824             LOG.error("Error adding VRF to datastore", e);
1825             throw new RuntimeException(e);
1826         }
1827     }
1828
1829     public synchronized void stopConfig() {
1830         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
1831                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
1832         InstanceIdentifier<ConfigServer> iid = iib.build();
1833         delete(iid);
1834     }
1835
1836     public synchronized void stopBgp() {
1837         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
1838                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
1839         InstanceIdentifier<AsId> iid = iib.build();
1840         delete(iid);
1841     }
1842
1843     public synchronized void delLogging() {
1844         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
1845                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
1846         InstanceIdentifier<Logging> iid = iib.build();
1847         delete(iid);
1848     }
1849
1850     public synchronized void delGracefulRestart() {
1851         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
1852                 InstanceIdentifier.builder(Bgp.class)
1853                         .child(GracefulRestart.class);
1854         InstanceIdentifier<GracefulRestart> iid = iib.build();
1855         delete(iid);
1856     }
1857
1858     public synchronized void delNeighbor(String nbrIp) {
1859         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1860         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
1861                 InstanceIdentifier.builder(Bgp.class)
1862                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
1863         InstanceIdentifier<Neighbors> iid = iib.build();
1864         delete(iid);
1865     }
1866
1867     public synchronized void delUpdateSource(String nbrIp) {
1868         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1869         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
1870                 InstanceIdentifier.builder(Bgp.class)
1871                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1872                         .child(UpdateSource.class);
1873         InstanceIdentifier<UpdateSource> iid = iib.build();
1874         delete(iid);
1875     }
1876
1877     public synchronized void delEbgpMultihop(String nbrIp) {
1878         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1879         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
1880                 InstanceIdentifier.builder(Bgp.class)
1881                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1882                         .child(EbgpMultihop.class);
1883         InstanceIdentifier<EbgpMultihop> iid = iib.build();
1884         delete(iid);
1885     }
1886
1887     public synchronized void delAddressFamily(String nbrIp, int afi, int safi) {
1888         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1889         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
1890                 InstanceIdentifier.builder(Bgp.class)
1891                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1892                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
1893         InstanceIdentifier<AddressFamilies> iid = iib.build();
1894         delete(iid);
1895     }
1896
1897     public synchronized void delPrefix(String rd, String pfx) {
1898         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
1899                 InstanceIdentifier.builder(Bgp.class)
1900                         .child(Networks.class, new NetworksKey(pfx, rd));
1901         InstanceIdentifier<Networks> iid = iib.build();
1902         delete(iid);
1903     }
1904
1905     public synchronized void delVrf(String rd) {
1906         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
1907                 InstanceIdentifier.builder(Bgp.class)
1908                         .child(Vrfs.class, new VrfsKey(rd));
1909         InstanceIdentifier<Vrfs> iid = iib.build();
1910         delete(iid);
1911     }
1912
1913     static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
1914
1915     /*
1916     * Remove Stale Marked Routes after timer expiry.
1917     */
1918     class RouteCleanup implements Callable<Integer> {
1919
1920         @Override
1921         public Integer call() {
1922             totalCleared = 0;
1923             try {
1924                 if (staledFibEntriesMap.isEmpty()) {
1925                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
1926                 } else {
1927                     for (String rd : staledFibEntriesMap.keySet()) {
1928                         if (Thread.interrupted()) {
1929                             return 0;
1930                         }
1931                         Map<String, String> map = staledFibEntriesMap.get(rd);
1932                         if (map != null) {
1933                             for (String prefix : map.keySet()) {
1934                                 if (Thread.interrupted()) {
1935                                     return 0;
1936                                 }
1937                                 totalCleared++;
1938                                 LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}", rd, prefix);
1939                                 fibDSWriter.removeFibEntryFromDS(rd, prefix);
1940                             }
1941                         }
1942                     }
1943                 }
1944             } finally {
1945                 staledFibEntriesMap.clear();
1946             }
1947             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
1948             return 0;
1949         }
1950     }
1951
1952     /*
1953      * BGP restart scenario, ODL-BGP manager was/is running.
1954      * On re-sync notification, Get a copy of FIB database.
1955      */
1956     public static void createStaleFibMap() {
1957         totalStaledCount = 0;
1958         try {
1959             /*
1960             * at the time Stale FIB creation, Wait till all PENDING write transaction
1961              * to complete (or)wait for max timeout value of STALE_FIB_WAIT Seconds.
1962              */
1963             int retry = STALE_FIB_WAIT;
1964             while ((BgpUtil.getGetPendingWrTransaction() != 0) && (retry > 0)) {
1965                 Thread.sleep(1000);
1966                 retry--;
1967                 if (retry == 0) {
1968                     LOG.error("TimeOut occured {} seconds, in waiting stale fibDSWriter create", STALE_FIB_WAIT);
1969                 }
1970             }
1971             staledFibEntriesMap.clear();
1972             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
1973             DataBroker db = BgpUtil.getBroker();
1974             if (db == null) {
1975                 LOG.error("Couldn't find BgpUtil dataBroker while creating createStaleFibMap");
1976                 return;
1977             }
1978
1979             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(BgpUtil.getBroker(),
1980                     LogicalDatastoreType.CONFIGURATION, id);
1981             if (fibEntries.isPresent()) {
1982                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
1983                 for (VrfTables vrfTable : staleVrfTables) {
1984                     Map<String, String> staleFibEntMap = new HashMap<>();
1985                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
1986                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
1987                             //Stale marking and cleanup is only meant for the routes learned through BGP.
1988                             continue;
1989                         }
1990                         if (Thread.interrupted()) {
1991                             break;
1992                         }
1993                         totalStaledCount++;
1994                         //Create MAP from staleVrfTables.
1995                         for (String nextHop : vrfEntry.getNextHopAddressList()) {
1996                             staleFibEntMap.put(vrfEntry.getDestPrefix(), nextHop + "/" + vrfEntry.getLabel());
1997                         }
1998                     }
1999                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), staleFibEntMap);
2000                 }
2001             } else {
2002                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
2003             }
2004         } catch (InterruptedException | ReadFailedException e) {
2005             LOG.error("createStaleFibMap:: error ", e);
2006         }
2007         LOG.error("created {} staled entries ", totalStaledCount);
2008     }
2009
2010     //map<rd, map<prefix/len, nexthop/label>>
2011     public static Map<String, Map<String, String>> getStaledFibEntriesMap() {
2012         return staledFibEntriesMap;
2013     }
2014
2015     //TODO: below function is for testing purpose with cli
2016     public static void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
2017         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
2018         fibDSWriter.removeFibEntryFromDS(rd, prefix + "/" + plen);
2019     }
2020
2021     public boolean isBgpConnected() {
2022         return bgpRouter.isBgpConnected();
2023     }
2024
2025     public long getLastConnectedTS() {
2026         return bgpRouter.getLastConnectedTS();
2027     }
2028
2029     public long getConnectTS() {
2030         return bgpRouter.getConnectTS();
2031     }
2032
2033     public long getStartTS() {
2034         return bgpRouter.getStartTS();
2035     }
2036
2037     public static int getTotalStaledCount() {
2038         return totalStaledCount;
2039     }
2040
2041     public static int getTotalCleared() {
2042         return totalCleared;
2043     }
2044
2045     public Timer getBgpCountersTimer() {
2046         return bgpCountersTimer;
2047     }
2048
2049     public BgpCounters getBgpCounters() {
2050         return bgpCounters;
2051     }
2052
2053     public void setBgpCountersTimer(Timer timer) {
2054         bgpCountersTimer = timer;
2055     }
2056
2057     public void setBgpAlarmsTimer(Timer timer) {
2058         bgpAlarmsTimer = timer;
2059     }
2060
2061     public void startBgpCountersTask() {
2062         if (getBgpCounters() == null) {
2063             bgpCounters = new BgpCounters(bgpConfigurationManager.getBgpSdncMipIp());
2064             setBgpCountersTimer(new Timer(true));
2065             getBgpCountersTimer().scheduleAtFixedRate(bgpCounters, 0, 120 * 1000);
2066             LOG.info("Bgp Counters task scheduled for every two minutes.");
2067
2068             bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
2069         }
2070     }
2071
2072     public void stopBgpCountersTask() {
2073         Timer timer = getBgpCountersTimer();
2074         if (getBgpCounters() != null) {
2075             timer.cancel();
2076             setBgpCountersTimer(null);
2077             bgpCounters = null;
2078         }
2079     }
2080
2081     public void startBgpAlarmsTask() {
2082         if (getBgpAlarms() == null) {
2083             bgpAlarms = new BgpAlarms(this);
2084             setBgpAlarmsTimer(new Timer(true));
2085             getBgpAlarmsTimer().scheduleAtFixedRate(bgpAlarms, 0, 60 * 1000);
2086             LOG.info("Bgp Alarms task scheduled for every minute.");
2087         }
2088     }
2089
2090     public void stopBgpAlarmsTask() {
2091         Timer timer = getBgpAlarmsTimer();
2092         if (getBgpAlarms() != null) {
2093             timer.cancel();
2094             setBgpAlarmsTimer(null);
2095             bgpAlarms = null;
2096         }
2097     }
2098
2099     public Timer getBgpAlarmsTimer() {
2100         return bgpAlarmsTimer;
2101     }
2102
2103     public BgpAlarms getBgpAlarms() {
2104         return bgpAlarms;
2105     }
2106
2107 }
2108