vpnservice/bgpmanager/bgpmanager-impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import com.google.common.base.Optional;
  11 import io.netty.util.concurrent.GlobalEventExecutor;
  12 import java.io.BufferedReader;
  13 import java.io.File;
  14 import java.io.FileNotFoundException;
  15 import java.io.FileReader;
  16 import java.io.IOException;
  17 import java.lang.reflect.Constructor;
  18 import java.lang.reflect.InvocationTargetException;
  19 import java.net.InetAddress;
  20 import java.net.NetworkInterface;
  21 import java.net.SocketException;
  22 import java.util.Collections;
  23 import java.util.Enumeration;
  24 import java.util.HashMap;
  25 import java.util.Iterator;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Timer;
  29 import java.util.TimerTask;
  30 import java.util.concurrent.Callable;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.CountDownLatch;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.concurrent.ExecutorService;
  35 import java.util.concurrent.Executors;
  36 import java.util.concurrent.Future;
  37 import java.util.concurrent.ScheduledExecutorService;
  38 import java.util.concurrent.TimeoutException;
  39 import java.util.concurrent.atomic.AtomicBoolean;
  40 import java.util.concurrent.atomic.AtomicInteger;
  41 import javax.annotation.Nullable;
  42
  43
  44 import org.apache.thrift.TException;
  45 import org.opendaylight.controller.config.api.osgi.WaitingServiceTracker;
  46 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  47 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  48 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  49 import org.opendaylight.controller.md.sal.common.api.clustering.CandidateAlreadyRegisteredException;
  50 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipService;
  51 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  52 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  53 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  54 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  55 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  56 import org.opendaylight.genius.mdsalutil.NwConstants;
  57 import org.opendaylight.genius.utils.batching.DefaultBatchHandler;
  58 import org.opendaylight.genius.utils.clustering.EntityOwnerUtils;
  59 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  60 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  61 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  62 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  63 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  64 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  65 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  66 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  67 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  68 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  69 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  70 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  71 import org.opendaylight.netvirt.bgpmanager.thrift.gen.protocol_type;
  72 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  73 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  74 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  75 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  76 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  77 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
  78 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EncapType;
  79 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.LayerType;
  80 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.TcpMd5SignaturePasswordType;
  81 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
  82 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
  84 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
  85 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
  86 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
  87 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
  88 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Multipath;
  90 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathKey;
  92 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Neighbors;
  93 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsKey;
  95 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Networks;
  96 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksKey;
  98 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpath;
  99 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathKey;
 101 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Vrfs;
 102 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsKey;
 104 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamilies;
 105 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesKey;
 107 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihop;
 108 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihopBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSource;
 110 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSourceBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.TcpSecurityOption;
 112 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOption;
 113 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOptionBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
 115 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 120 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 121 import org.opendaylight.yangtools.yang.binding.DataObject;
 122 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 123 import org.osgi.framework.BundleContext;
 124 import org.slf4j.Logger;
 125 import org.slf4j.LoggerFactory;
 126
 127
 128
 129 public class BgpConfigurationManager {
 130     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 131     private static DataBroker dataBroker;
 132     private static FibDSWriter fibDSWriter;
 133     public static IBgpManager bgpManager;
 134     private static IVpnLinkService vpnLinkService;
 135     private final BundleContext bundleContext;
 136     private static Bgp config;
 137     private static BgpRouter bgpRouter;
 138     private static BgpThriftService updateServer;
 139     private BgpCounters bgpCounters;
 140     private BgpAlarms bgpAlarms;
 141     private Timer bgpCountersTimer;
 142     private Timer bgpAlarmsTimer;
 143     private static final String DEF_LOGFILE = "/var/log/bgp_debug.log";
 144     private static final String DEF_LOGLEVEL = "errors";
 145     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 146     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 147     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 148     private static final String DEF_UPORT = "6644";
 149     private static final String DEF_CHOST = "127.0.0.1";
 150     private static final String DEF_CPORT = "7644";
 151     private static final String DEF_SDNC_BGP_MIP = "127.0.0.1";
 152     private static final String DEF_BGP_SDNC_MIP = "127.0.0.1";
 153     private static final String SDNC_BGP_MIP = "vpnservice.bgp.thrift.bgp.mip";
 154     private static final String BGP_SDNC_MIP = "vpnservice.bgp.thrift.sdnc.mip";
 155     private static final String CLUSTER_CONF_FILE = "/cluster/etc/cluster.conf";
 156     private static final Timer IP_ACTIVATION_CHECK_TIMER = new Timer();
 157     private static final int STALE_FIB_WAIT = 60;
 158     private static final int RESTART_DEFAULT_GR = 90;
 159     private long staleStartTime = 0;
 160     private long staleEndTime = 0;
 161     private long cfgReplayStartTime = 0;
 162     private long cfgReplayEndTime = 0;
 163     private long staleCleanupTime = 0;
 164     private static final int DS_RETRY_COOUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
 165     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
 166
 167     public String getBgpSdncMipIp() {
 168         return getProperty(BGP_SDNC_MIP, DEF_BGP_SDNC_MIP);
 169     }
 170
 171     public long getStaleCleanupTime() {
 172         return staleCleanupTime;
 173     }
 174
 175     public void setStaleCleanupTime(long staleCleanupTime) {
 176         this.staleCleanupTime = staleCleanupTime;
 177     }
 178
 179     public long getCfgReplayEndTime() {
 180         return cfgReplayEndTime;
 181     }
 182
 183     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 184         this.cfgReplayEndTime = cfgReplayEndTime;
 185     }
 186
 187     public long getCfgReplayStartTime() {
 188         return cfgReplayStartTime;
 189     }
 190
 191     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 192         this.cfgReplayStartTime = cfgReplayStartTime;
 193     }
 194
 195     public long getStaleEndTime() {
 196         return staleEndTime;
 197     }
 198
 199     public void setStaleEndTime(long staleEndTime) {
 200         this.staleEndTime = staleEndTime;
 201     }
 202
 203     public long getStaleStartTime() {
 204         return staleStartTime;
 205     }
 206
 207     public void setStaleStartTime(long staleStartTime) {
 208         this.staleStartTime = staleStartTime;
 209     }
 210
 211
 212     // to have stale FIB map (RD, Prefix)
 213     //  number of seconds wait for route sync-up between ODL and BGP
 214     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 215
 216     static String odlThriftIp = "127.0.0.1";
 217     static String bgpThriftIp = "127.0.0.1";
 218     private static String cHostStartup;
 219     private static String cPortStartup;
 220     private static CountDownLatch initer = new CountDownLatch(1);
 221     //static IITMProvider itmProvider;
 222     //map<rd, map<prefix/len:nexthop, label>>
 223     private static Map<String, Map<String, Long>> staledFibEntriesMap = new ConcurrentHashMap<>();
 224
 225     static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 226     static final String BGP_ENTITY_NAME = "bgp";
 227
 228     static int totalStaledCount = 0;
 229     static int totalCleared = 0;
 230     static int totalExternalRoutes = 0;
 231
 232     private static final Class[] REACTORS = {
 233         ConfigServerReactor.class, AsIdReactor.class,
 234         GracefulRestartReactor.class, LoggingReactor.class,
 235         NeighborsReactor.class, UpdateSourceReactor.class,
 236         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 237         NetworksReactor.class, VrfsReactor.class, BgpReactor.class,
 238         MultipathReactor.class, VrfMaxpathReactor.class
 239     };
 240
 241     private ListenerRegistration<DataChangeListener>[] registrations;
 242
 243     final BgpConfigurationManager bgpConfigurationManager;
 244
 245     public BgpConfigurationManager(final DataBroker dataBroker,
 246             final EntityOwnershipService entityOwnershipService,
 247             final FibDSWriter fibDSWriter,
 248             final IVpnLinkService vpnLinkSrvce,
 249             final BundleContext bundleContext)
 250             throws InterruptedException, ExecutionException, TimeoutException {
 251         BgpConfigurationManager.dataBroker = dataBroker;
 252         BgpConfigurationManager.fibDSWriter = fibDSWriter;
 253         BgpConfigurationManager.vpnLinkService = vpnLinkService;
 254         this.bundleContext = bundleContext;
 255         String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
 256         cHostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 257         cPortStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 258         LOG.info("UpdateServer at localhost:" + updatePort + " ConfigServer at "
 259                 + cHostStartup + ":" + cPortStartup);
 260         VtyshCli.setHostAddr(cHostStartup);
 261         ClearBgpCli.setHostAddr(cHostStartup);
 262         setEntityOwnershipService(entityOwnershipService);
 263         bgpRouter = BgpRouter.getInstance();
 264         odlThriftIp = getProperty(SDNC_BGP_MIP, DEF_SDNC_BGP_MIP);
 265         bgpThriftIp = getProperty(BGP_SDNC_MIP, DEF_BGP_SDNC_MIP);
 266         registerCallbacks();
 267
 268         LOG.info("BGP Configuration manager initialized");
 269         initer.countDown();
 270
 271         bgpConfigurationManager = this;
 272         BgpUtil.batchSize = BgpUtil.BATCH_SIZE;
 273         if (Integer.getInteger("batch.size") != null) {
 274             BgpUtil.batchSize = Integer.getInteger("batch.size");
 275         }
 276         BgpUtil.batchInterval = BgpUtil.PERIODICITY;
 277         if (Integer.getInteger("batch.wait.time") != null) {
 278             BgpUtil.batchInterval = Integer.getInteger("batch.wait.time");
 279         }
 280         BgpUtil.registerWithBatchManager(
 281                 new DefaultBatchHandler(dataBroker, LogicalDatastoreType.CONFIGURATION, BgpUtil.batchSize,
 282                         BgpUtil.batchInterval));
 283
 284         GlobalEventExecutor.INSTANCE.execute(() -> {
 285             final WaitingServiceTracker<IBgpManager> tracker = WaitingServiceTracker.create(
 286                     IBgpManager.class, bundleContext);
 287             bgpManager = tracker.waitForService(WaitingServiceTracker.FIVE_MINUTES);
 288             updateServer = new BgpThriftService(Integer.parseInt(updatePort), bgpManager, fibDSWriter);
 289             updateServer.start();
 290             LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 291         });
 292     }
 293
 294     private Object createListener(Class<?> cls) {
 295         Constructor<?> ctor;
 296         Object obj = null;
 297
 298         try {
 299             ctor = cls.getConstructor(BgpConfigurationManager.class);
 300             obj = ctor.newInstance(this);
 301         } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException
 302                 e) {
 303             LOG.error("Failed to create listener object", e);
 304         }
 305         return obj;
 306     }
 307
 308     private void registerCallbacks() {
 309         String emsg = "Failed to register listener";
 310         registrations = new ListenerRegistration[REACTORS.length];
 311         InstanceIdentifier<?> iid = InstanceIdentifier.create(Bgp.class);
 312         for (Class reactor : REACTORS) {
 313             Object obj = createListener(reactor);
 314             AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 315             dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 316         }
 317     }
 318
 319     public void close() {
 320         if (updateServer != null) {
 321             updateServer.stop();
 322         }
 323         LOG.info("{} close", getClass().getSimpleName());
 324     }
 325
 326     private boolean configExists() throws ReadFailedException {
 327         InstanceIdentifier.InstanceIdentifierBuilder<Bgp> iib =
 328                 InstanceIdentifier.builder(Bgp.class);
 329         InstanceIdentifier<Bgp> iid = iib.build();
 330         return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 331                 iid).isPresent();
 332     }
 333
 334     private String getProperty(String var, String def) {
 335         String property = bundleContext.getProperty(var);
 336         return (property == null ? def : property);
 337     }
 338
 339     boolean ignoreClusterDcnEventForFollower() {
 340         return !EntityOwnerUtils.amIEntityOwner(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME);
 341     }
 342
 343     public Bgp get() {
 344         config = getConfig();
 345         return config;
 346     }
 347
 348     public void setEntityOwnershipService(final EntityOwnershipService entityOwnershipService) {
 349         try {
 350             EntityOwnerUtils.registerEntityCandidateForOwnerShip(entityOwnershipService,
 351                 BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME, ownershipChange -> {
 352                     LOG.trace("entity owner change event fired");
 353                     if (ownershipChange.hasOwner() && ownershipChange.isOwner()) {
 354                         LOG.trace("This PL is the Owner");
 355                         activateMIP();
 356                         bgpRestarted();
 357                     } else {
 358                         LOG.info("Not owner: hasOwner: {}, isOwner: {}", ownershipChange.hasOwner(),
 359                                 ownershipChange.isOwner());
 360                     }
 361                 });
 362         } catch (CandidateAlreadyRegisteredException e) {
 363             LOG.error("failed to register bgp entity", e);
 364         }
 365     }
 366
 367     private static final String ADD_WARN =
 368             "Config store updated; undo with Delete if needed.";
 369     private static final String DEL_WARN =
 370             "Config store updated; undo with Add if needed.";
 371     private static final String UPD_WARN =
 372             "Update operation not supported; Config store updated;"
 373                     + " restore with another Update if needed.";
 374
 375     public class ConfigServerReactor
 376             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 377             implements AutoCloseable, ClusteredDataTreeChangeListener<ConfigServer> {
 378         private static final String YANG_OBJ = "config-server ";
 379
 380         public ConfigServerReactor() {
 381             super(ConfigServer.class, ConfigServerReactor.class);
 382         }
 383
 384         @Override
 385         protected synchronized void add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 386             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 387             if (ignoreClusterDcnEventForFollower()) {
 388                 return;
 389             }
 390
 391             try {
 392                 initer.await();
 393             } catch (InterruptedException e) {
 394                 // Ignored
 395             }
 396             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 397             synchronized (BgpConfigurationManager.this) {
 398                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 399                         val.getPort().intValue());
 400                 if (!res) {
 401                     LOG.error(YANG_OBJ + "Add failed; " + ADD_WARN);
 402                 }
 403             }
 404         }
 405
 406         @Override
 407         protected ConfigServerReactor getDataTreeChangeListener() {
 408             return ConfigServerReactor.this;
 409         }
 410
 411         @Override
 412         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 413             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 414         }
 415
 416         @Override
 417         protected synchronized void remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 418             LOG.trace("received bgp disconnect");
 419             if (ignoreClusterDcnEventForFollower()) {
 420                 return;
 421             }
 422             synchronized (BgpConfigurationManager.this) {
 423                 bgpRouter.disconnect();
 424             }
 425         }
 426
 427         @Override
 428         protected void update(InstanceIdentifier<ConfigServer> iid,
 429                 ConfigServer oldval, ConfigServer newval) {
 430             LOG.trace("received bgp Connection update");
 431             if (ignoreClusterDcnEventForFollower()) {
 432                 return;
 433             }
 434             LOG.error(YANG_OBJ + UPD_WARN);
 435         }
 436     }
 437
 438     private BgpRouter getClient(String yangObj) {
 439         if (bgpRouter == null || !bgpRouter.isBgpConnected()) {
 440             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 441             return null;
 442         }
 443         return bgpRouter;
 444     }
 445
 446     public class AsIdReactor
 447             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 448             implements AutoCloseable, ClusteredDataTreeChangeListener<AsId> {
 449
 450         private static final String YANG_OBJ = "as-id ";
 451
 452         public AsIdReactor() {
 453             super(AsId.class, AsIdReactor.class);
 454         }
 455
 456         @Override
 457         protected synchronized void add(InstanceIdentifier<AsId> iid, AsId val) {
 458             LOG.error("received bgp add asid {}", val);
 459             if (ignoreClusterDcnEventForFollower()) {
 460                 return;
 461             }
 462             LOG.debug("received add router config asNum {}", val.getLocalAs());
 463             synchronized (BgpConfigurationManager.this) {
 464                 BgpRouter br = getClient(YANG_OBJ);
 465                 if (br == null) {
 466                     LOG.error("{} Unable to process add for asNum {}; {}", YANG_OBJ, val.getLocalAs(),
 467                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 468                     return;
 469                 }
 470                 if (isIpAvailable(odlThriftIp)) {
 471                     bgpRestarted();
 472                 } else {
 473                     IP_ACTIVATION_CHECK_TIMER.scheduleAtFixedRate(new TimerTask() {
 474                         @Override
 475                         public void run() {
 476                             if (isIpAvailable(odlThriftIp)) {
 477                                 bgpRestarted();
 478                                 IP_ACTIVATION_CHECK_TIMER.cancel();
 479                             } else {
 480                                 LOG.trace("waiting for odlThriftIP: {} to be present", odlThriftIp);
 481                             }
 482                         }
 483                     }, 10000L, 10000L);
 484                 }
 485                 if (getBgpCounters() == null) {
 486                     startBgpCountersTask();
 487                 }
 488                 if (getBgpAlarms() == null) {
 489                     startBgpAlarmsTask();
 490                 }
 491             }
 492         }
 493
 494         @Override
 495         protected AsIdReactor getDataTreeChangeListener() {
 496             return AsIdReactor.this;
 497         }
 498
 499         @Override
 500         protected InstanceIdentifier<AsId> getWildCardPath() {
 501             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 502         }
 503
 504         @Override
 505         protected synchronized void remove(InstanceIdentifier<AsId> iid, AsId val) {
 506             LOG.error("received delete router config asNum {}", val.getLocalAs());
 507             if (ignoreClusterDcnEventForFollower()) {
 508                 return;
 509             }
 510             synchronized (BgpConfigurationManager.this) {
 511                 long asNum = val.getLocalAs();
 512                 BgpRouter br = getClient(YANG_OBJ);
 513                 if (br == null) {
 514                     LOG.error("{} Unable to process remove for asNum {}; {}", YANG_OBJ, asNum,
 515                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 516                     return;
 517                 }
 518                 try {
 519                     br.stopBgp(asNum);
 520                 } catch (TException | BgpRouterException e) {
 521                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 522                 }
 523                 if (getBgpCounters() != null) {
 524                     stopBgpCountersTask();
 525                 }
 526                 if (getBgpAlarms() != null) {
 527                     stopBgpAlarmsTask();
 528                 }
 529                 Bgp conf = getConfig();
 530                 if (conf == null) {
 531                     LOG.error("Config Null while removing the as-id");
 532                     return;
 533                 }
 534                 LOG.debug("Removing external routes from FIB");
 535                 deleteExternalFibRoutes();
 536                 List<Neighbors> nbrs = conf.getNeighbors();
 537                 if (nbrs != null && nbrs.size() > 0) {
 538                     LOG.error("Tring to remove the as-id when neighbor config is already present");
 539                     for (Neighbors nbr : nbrs) {
 540                         LOG.debug("Removing Neighbor {} from Data store", nbr.getAddress().getValue());
 541                         delNeighbor(nbr.getAddress().getValue());
 542                     }
 543                 }
 544             }
 545         }
 546
 547         @Override
 548         protected void update(InstanceIdentifier<AsId> iid,
 549                 AsId oldval, AsId newval) {
 550             if (ignoreClusterDcnEventForFollower()) {
 551                 return;
 552             }
 553             LOG.error(YANG_OBJ + UPD_WARN);
 554         }
 555     }
 556
 557     public class GracefulRestartReactor
 558             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 559             implements AutoCloseable, ClusteredDataTreeChangeListener<GracefulRestart> {
 560
 561         private static final String YANG_OBJ = "graceful-restart ";
 562
 563         public GracefulRestartReactor() {
 564             super(GracefulRestart.class, GracefulRestartReactor.class);
 565         }
 566
 567         @Override
 568         protected synchronized void add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 569             if (ignoreClusterDcnEventForFollower()) {
 570                 return;
 571             }
 572             synchronized (BgpConfigurationManager.this) {
 573                 int stalePathTime = val.getStalepathTime().intValue();
 574                 BgpRouter br = getClient(YANG_OBJ);
 575                 if (br == null) {
 576                     LOG.error("{} Unable to add stale-path time {}; {}", YANG_OBJ, stalePathTime,
 577                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 578                     return;
 579                 }
 580                 try {
 581                     br.addGracefulRestart(stalePathTime);
 582                 } catch (TException | BgpRouterException e) {
 583                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 584                 }
 585             }
 586         }
 587
 588         @Override
 589         protected GracefulRestartReactor getDataTreeChangeListener() {
 590             return GracefulRestartReactor.this;
 591         }
 592
 593         @Override
 594         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 595             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 596         }
 597
 598         @Override
 599         protected synchronized void remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 600             if (ignoreClusterDcnEventForFollower()) {
 601                 return;
 602             }
 603             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 604             synchronized (BgpConfigurationManager.this) {
 605                 BgpRouter br = getClient(YANG_OBJ);
 606                 if (br == null) {
 607                     LOG.error("{} Unable to delete stale-path time; {}", YANG_OBJ,
 608                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 609                     return;
 610                 }
 611                 try {
 612                     br.delGracefulRestart();
 613                 } catch (TException | BgpRouterException e) {
 614                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 615                 }
 616             }
 617         }
 618
 619         @Override
 620         protected void update(InstanceIdentifier<GracefulRestart> iid,
 621                 GracefulRestart oldval, GracefulRestart newval) {
 622             if (ignoreClusterDcnEventForFollower()) {
 623                 return;
 624             }
 625             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 626             synchronized (BgpConfigurationManager.this) {
 627                 int stalePathTime = newval.getStalepathTime().intValue();
 628                 BgpRouter br = getClient(YANG_OBJ);
 629                 if (br == null) {
 630                     LOG.error("{} Unable to update stale-path time to {}; {}", YANG_OBJ, stalePathTime,
 631                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 632                     return;
 633                 }
 634                 try {
 635                     br.addGracefulRestart(stalePathTime);
 636                 } catch (TException | BgpRouterException e) {
 637                     LOG.error("{} update received exception; {}", YANG_OBJ, ADD_WARN, e);
 638                 }
 639             }
 640         }
 641     }
 642
 643     public class LoggingReactor
 644             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 645             implements AutoCloseable, ClusteredDataTreeChangeListener<Logging> {
 646
 647         private static final String YANG_OBJ = "logging ";
 648
 649         public LoggingReactor() {
 650             super(Logging.class, LoggingReactor.class);
 651         }
 652
 653         @Override
 654         protected synchronized void add(InstanceIdentifier<Logging> iid, Logging val) {
 655             if (ignoreClusterDcnEventForFollower()) {
 656                 return;
 657             }
 658             synchronized (BgpConfigurationManager.this) {
 659                 BgpRouter br = getClient(YANG_OBJ);
 660                 if (br == null) {
 661                     LOG.error("{} Unable to add logging for qbgp; {}", YANG_OBJ,
 662                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 663                     return;
 664                 }
 665                 try {
 666                     br.setLogging(val.getFile(), val.getLevel());
 667                 } catch (TException | BgpRouterException e) {
 668                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 669                 }
 670             }
 671         }
 672
 673         @Override
 674         protected LoggingReactor getDataTreeChangeListener() {
 675             return LoggingReactor.this;
 676         }
 677
 678         @Override
 679         protected InstanceIdentifier<Logging> getWildCardPath() {
 680             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 681         }
 682
 683         @Override
 684         protected synchronized void remove(InstanceIdentifier<Logging> iid, Logging val) {
 685             if (ignoreClusterDcnEventForFollower()) {
 686                 return;
 687             }
 688             LOG.debug("received remove Logging config val {}", val.getLevel());
 689             synchronized (BgpConfigurationManager.this) {
 690                 BgpRouter br = getClient(YANG_OBJ);
 691                 if (br == null) {
 692                     LOG.error("{} Unable to remove logging for qbgp; {}", YANG_OBJ,
 693                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 694                     return;
 695                 }
 696                 try {
 697                     br.setLogging(DEF_LOGFILE, DEF_LOGLEVEL);
 698                 } catch (TException | BgpRouterException e) {
 699                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 700                 }
 701             }
 702         }
 703
 704         @Override
 705         protected void update(InstanceIdentifier<Logging> iid,
 706                 Logging oldval, Logging newval) {
 707             if (ignoreClusterDcnEventForFollower()) {
 708                 return;
 709             }
 710             synchronized (BgpConfigurationManager.this) {
 711                 BgpRouter br = getClient(YANG_OBJ);
 712                 if (br == null) {
 713                     LOG.error("{} Unable to update logging for qbgp; {}", YANG_OBJ,
 714                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 715                     return;
 716                 }
 717                 try {
 718                     br.setLogging(newval.getFile(), newval.getLevel());
 719                 } catch (TException | BgpRouterException e) {
 720                     LOG.error("{} newval received exception; {}", YANG_OBJ, ADD_WARN, e);
 721                 }
 722             }
 723         }
 724     }
 725
 726     public class NeighborsReactor
 727             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 728             implements AutoCloseable, ClusteredDataTreeChangeListener<Neighbors> {
 729
 730         private static final String YANG_OBJ = "neighbors ";
 731
 732         public NeighborsReactor() {
 733             super(Neighbors.class, NeighborsReactor.class);
 734         }
 735
 736         @Override
 737         protected synchronized void add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 738             if (ignoreClusterDcnEventForFollower()) {
 739                 return;
 740             }
 741             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 742             synchronized (BgpConfigurationManager.this) {
 743                 String peerIp = val.getAddress().getValue();
 744                 long as = val.getRemoteAs();
 745                 final String md5Secret = extractMd5Secret(val);
 746                 BgpRouter br = getClient(YANG_OBJ);
 747                 if (br == null) {
 748                     LOG.error("{} Unable to process add for peer {} as {}; {}", YANG_OBJ, peerIp, as,
 749                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 750                     return;
 751                 }
 752                 try {
 753                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 754                     br.addNeighbor(peerIp, as, md5Secret);
 755
 756                 } catch (TException | BgpRouterException e) {
 757                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 758                 }
 759             }
 760         }
 761
 762         @Override
 763         protected NeighborsReactor getDataTreeChangeListener() {
 764             return NeighborsReactor.this;
 765         }
 766
 767         @Override
 768         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 769             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class);
 770         }
 771
 772         @Override
 773         protected synchronized void remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 774             if (ignoreClusterDcnEventForFollower()) {
 775                 return;
 776             }
 777             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 778             synchronized (BgpConfigurationManager.this) {
 779                 String peerIp = val.getAddress().getValue();
 780                 BgpRouter br = getClient(YANG_OBJ);
 781                 if (br == null) {
 782                     LOG.error("{} Unable to process remove for peer {}; {}", YANG_OBJ, peerIp,
 783                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 784                     return;
 785                 }
 786                 try {
 787                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 788                     br.delNeighbor(peerIp);
 789                 } catch (TException | BgpRouterException e) {
 790                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 791                 }
 792             }
 793         }
 794
 795         @Override
 796         protected void update(InstanceIdentifier<Neighbors> iid,
 797                 Neighbors oldval, Neighbors newval) {
 798             if (ignoreClusterDcnEventForFollower()) {
 799                 return;
 800             }
 801             //purposefully nothing to do.
 802         }
 803     }
 804
 805     public class EbgpMultihopReactor
 806             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 807             implements AutoCloseable, ClusteredDataTreeChangeListener<EbgpMultihop> {
 808
 809         private static final String YANG_OBJ = "ebgp-multihop ";
 810
 811         public EbgpMultihopReactor() {
 812             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 813         }
 814
 815         @Override
 816         protected synchronized void add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 817             if (ignoreClusterDcnEventForFollower()) {
 818                 return;
 819             }
 820             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 821             synchronized (BgpConfigurationManager.this) {
 822                 String peerIp = val.getPeerIp().getValue();
 823                 BgpRouter br = getClient(YANG_OBJ);
 824                 if (br == null) {
 825                     LOG.error("{} Unable to process add for peer {}; {}", YANG_OBJ, peerIp,
 826                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 827                     return;
 828                 }
 829                 try {
 830                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 831                 } catch (TException | BgpRouterException e) {
 832                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 833                 }
 834             }
 835         }
 836
 837         @Override
 838         protected EbgpMultihopReactor getDataTreeChangeListener() {
 839             return EbgpMultihopReactor.this;
 840         }
 841
 842         @Override
 843         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
 844             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(EbgpMultihop.class);
 845         }
 846
 847         @Override
 848         protected synchronized void remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 849             if (ignoreClusterDcnEventForFollower()) {
 850                 return;
 851             }
 852             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 853             synchronized (BgpConfigurationManager.this) {
 854                 String peerIp = val.getPeerIp().getValue();
 855                 BgpRouter br = getClient(YANG_OBJ);
 856                 if (br == null) {
 857                     LOG.error("{} Unable to process remove for peer {}; {}", YANG_OBJ, peerIp,
 858                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 859                     return;
 860                 }
 861                 try {
 862                     br.delEbgpMultihop(peerIp);
 863                 } catch (TException | BgpRouterException e) {
 864                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 865                 }
 866             }
 867         }
 868
 869         @Override
 870         protected void update(InstanceIdentifier<EbgpMultihop> iid,
 871                 EbgpMultihop oldval, EbgpMultihop newval) {
 872             if (ignoreClusterDcnEventForFollower()) {
 873                 return;
 874             }
 875             LOG.error(YANG_OBJ + UPD_WARN);
 876         }
 877     }
 878
 879     public class UpdateSourceReactor
 880             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
 881             implements AutoCloseable, ClusteredDataTreeChangeListener<UpdateSource> {
 882
 883         private static final String YANG_OBJ = "update-source ";
 884
 885         public UpdateSourceReactor() {
 886             super(UpdateSource.class, UpdateSourceReactor.class);
 887         }
 888
 889         @Override
 890         protected synchronized void add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 891             if (ignoreClusterDcnEventForFollower()) {
 892                 return;
 893             }
 894             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
 895             synchronized (BgpConfigurationManager.this) {
 896                 String peerIp = val.getPeerIp().getValue();
 897                 BgpRouter br = getClient(YANG_OBJ);
 898                 if (br == null) {
 899                     LOG.error("{} Unable to process add for peer {}; {}", YANG_OBJ, peerIp,
 900                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 901                     return;
 902                 }
 903                 try {
 904                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
 905                 } catch (TException | BgpRouterException e) {
 906                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 907                 }
 908             }
 909         }
 910
 911         @Override
 912         protected UpdateSourceReactor getDataTreeChangeListener() {
 913             return UpdateSourceReactor.this;
 914         }
 915
 916         @Override
 917         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
 918             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(UpdateSource.class);
 919         }
 920
 921         @Override
 922         protected synchronized void remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 923             if (ignoreClusterDcnEventForFollower()) {
 924                 return;
 925             }
 926             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
 927             synchronized (BgpConfigurationManager.this) {
 928                 String peerIp = val.getPeerIp().getValue();
 929                 BgpRouter br = getClient(YANG_OBJ);
 930                 if (br == null) {
 931                     LOG.error("{} Unable to process remove for peer {}; {}", YANG_OBJ, peerIp,
 932                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 933                     return;
 934                 }
 935                 try {
 936                     br.delUpdateSource(peerIp);
 937                 } catch (TException | BgpRouterException e) {
 938                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 939                 }
 940             }
 941         }
 942
 943         @Override
 944         protected void update(InstanceIdentifier<UpdateSource> iid,
 945                 UpdateSource oldval, UpdateSource newval) {
 946             if (ignoreClusterDcnEventForFollower()) {
 947                 return;
 948             }
 949             LOG.error(YANG_OBJ + UPD_WARN);
 950         }
 951     }
 952
 953     public class AddressFamiliesReactor
 954             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
 955             implements AutoCloseable, ClusteredDataTreeChangeListener<AddressFamilies> {
 956
 957         private static final String YANG_OBJ = "address-families ";
 958
 959         public AddressFamiliesReactor() {
 960             super(AddressFamilies.class, AddressFamiliesReactor.class);
 961         }
 962
 963         @Override
 964         protected synchronized void add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
 965             if (ignoreClusterDcnEventForFollower()) {
 966                 return;
 967             }
 968             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
 969             synchronized (BgpConfigurationManager.this) {
 970                 String peerIp = val.getPeerIp().getValue();
 971                 BgpRouter br = getClient(YANG_OBJ);
 972                 if (br == null) {
 973                     LOG.error("{} Unable to process add for peer {}; {}", YANG_OBJ, peerIp,
 974                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 975                     return;
 976                 }
 977                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
 978                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
 979                 try {
 980                     br.addAddressFamily(peerIp, afi, safi);
 981                 } catch (TException | BgpRouterException e) {
 982                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 983                 }
 984             }
 985         }
 986
 987         @Override
 988         protected AddressFamiliesReactor getDataTreeChangeListener() {
 989             return AddressFamiliesReactor.this;
 990         }
 991
 992         @Override
 993         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
 994             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(AddressFamilies.class);
 995         }
 996
 997         @Override
 998         protected synchronized void remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
 999             if (ignoreClusterDcnEventForFollower()) {
1000                 return;
1001             }
1002             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
1003             synchronized (BgpConfigurationManager.this) {
1004                 String peerIp = val.getPeerIp().getValue();
1005                 BgpRouter br = getClient(YANG_OBJ);
1006                 if (br == null) {
1007                     LOG.error("{} Unable to process remove for peer {}; {}", YANG_OBJ, peerIp,
1008                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1009                     return;
1010                 }
1011                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1012                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1013                 try {
1014                     br.delAddressFamily(peerIp, afi, safi);
1015                 } catch (TException | BgpRouterException e) {
1016                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1017                 }
1018             }
1019         }
1020
1021         @Override
1022         protected void update(InstanceIdentifier<AddressFamilies> iid,
1023                 AddressFamilies oldval, AddressFamilies newval) {
1024             if (ignoreClusterDcnEventForFollower()) {
1025                 return;
1026             }
1027             LOG.error(YANG_OBJ + UPD_WARN);
1028         }
1029     }
1030
1031     public class NetworksReactor
1032             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
1033             implements AutoCloseable, ClusteredDataTreeChangeListener<Networks> {
1034
1035         private static final String YANG_OBJ = "networks ";
1036
1037         public NetworksReactor() {
1038             super(Networks.class, NetworksReactor.class);
1039         }
1040
1041         @Override
1042         public NetworksReactor getDataTreeChangeListener() {
1043             return NetworksReactor.this;
1044         }
1045
1046         @Override
1047         protected synchronized void add(InstanceIdentifier<Networks> iid, Networks val) {
1048             if (ignoreClusterDcnEventForFollower()) {
1049                 return;
1050             }
1051             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1052             synchronized (BgpConfigurationManager.this) {
1053                 String rd = val.getRd();
1054                 String pfxlen = val.getPrefixLen();
1055                 String nh = val.getNexthop().getValue();
1056                 BgpRouter br = getClient(YANG_OBJ);
1057                 if (br == null) {
1058                     LOG.error("{} Unable to process add for rd {} prefix {} nexthop {}; {}", YANG_OBJ, rd, pfxlen, nh,
1059                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1060                     return;
1061                 }
1062                 Long label = val.getLabel();
1063                 int lbl = (label == null) ? qbgpConstants.LBL_NO_LABEL
1064                         : label.intValue();
1065                 int l3vni = (val.getL3vni() == null) ? qbgpConstants.LBL_NO_LABEL
1066                         : val.getL3vni().intValue();
1067
1068                 BgpControlPlaneType protocolType = val.getBgpControlPlaneType();
1069                 int ethernetTag = val.getEthtag().intValue();
1070                 String esi = val.getEsi();
1071                 String macaddress = val.getMacaddress();
1072                 EncapType encapType = val.getEncapType();
1073                 String routerMac = val.getRoutermac();
1074                 int afiInt = testValueAFI(pfxlen);
1075
1076                 try {
1077                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, BgpUtil.convertToThriftProtocolType(protocolType),
1078                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType),
1079                             routerMac);
1080                 } catch (TException | BgpRouterException e) {
1081                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1082                 }
1083             }
1084         }
1085
1086         @Override
1087         protected InstanceIdentifier<Networks> getWildCardPath() {
1088             return InstanceIdentifier.create(Bgp.class).child(Networks.class);
1089         }
1090
1091         @Override
1092         protected synchronized void remove(InstanceIdentifier<Networks> iid, Networks val) {
1093             if (ignoreClusterDcnEventForFollower()) {
1094                 return;
1095             }
1096             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1097             synchronized (BgpConfigurationManager.this) {
1098                 String rd = val.getRd();
1099                 String pfxlen = val.getPrefixLen();
1100                 BgpRouter br = getClient(YANG_OBJ);
1101                 if (br == null) {
1102                     LOG.error("{} Unable to process remove for rd {} prefix {}; {}", YANG_OBJ, rd, pfxlen,
1103                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1104                     return;
1105                 }
1106                 Long label = val.getLabel();
1107                 int lbl = (label == null) ? 0 : label.intValue();
1108                 int  afiInt = testValueAFI(pfxlen);
1109                 if (rd == null && lbl > 0) {
1110                     //LU prefix is being deleted.
1111                     rd = Integer.toString(lbl);
1112                 }
1113                 try {
1114                     br.delPrefix(rd, pfxlen);
1115                 } catch (TException | BgpRouterException e) {
1116                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1117                 }
1118             }
1119         }
1120
1121         /**get the value AFI from a prefix as "x.x.x.x/x".
1122          *
1123          * @param pfxlen the prefix to get an afi
1124          * @return the afi value as you are need
1125          */
1126         public  int  testValueAFI(String pfxlen) {
1127             int afiNew = af_afi.AFI_IP.getValue();
1128             try {
1129                 String ipOnly = pfxlen.substring(0, pfxlen.lastIndexOf("/"));
1130                 java.net.Inet6Address.getByName(ipOnly);
1131                 afiNew = af_afi.AFI_IPV6.getValue();
1132             } catch (java.net.UnknownHostException e) {
1133                 //ce n'est pas de l'ipv6
1134             }
1135             return afiNew;
1136         }
1137
1138
1139         @Override
1140         protected void update(final InstanceIdentifier<Networks> iid,
1141                 final Networks oldval, final Networks newval) {
1142             if (ignoreClusterDcnEventForFollower()) {
1143                 return;
1144             }
1145             if (oldval.equals(newval)) {
1146                 //Update: OLD and New values are same, no need to trigger remove/add.
1147                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1148                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1149                 return;
1150             }
1151             LOG.debug("received update networks config val {}", newval.getPrefixLen());
1152             remove(iid, oldval);
1153             timer.schedule(new TimerTask() {
1154                 @Override
1155                 public void run() {
1156                     add(iid, newval);
1157                 }
1158             }, Integer.getInteger("bgp.nexthop.update.delay.in.secs", 5) * 1000);
1159         }
1160     }
1161
1162     static Timer timer = new Timer();
1163
1164     public class VrfsReactor
1165             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1166             implements AutoCloseable, ClusteredDataTreeChangeListener<Vrfs> {
1167
1168         private static final String YANG_OBJ = "vrfs ";
1169
1170         public VrfsReactor() {
1171             super(Vrfs.class, VrfsReactor.class);
1172         }
1173
1174         @Override
1175         protected synchronized void add(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1176             if (ignoreClusterDcnEventForFollower()) {
1177                 return;
1178             }
1179             LOG.debug("received add Vrfs config val {}", val.getRd());
1180             synchronized (BgpConfigurationManager.this) {
1181                 String rd = val.getRd();
1182                 BgpRouter br = getClient(YANG_OBJ);
1183                 if (br == null) {
1184                     LOG.error("{} Unable to process add for rd {}; {}", YANG_OBJ, rd,
1185                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1186                     return;
1187                 }
1188                 try {
1189                     br.addVrf(val.getLayerType(), rd, val.getImportRts(),
1190                             val.getExportRts());
1191                 } catch (TException | BgpRouterException e) {
1192                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1193                 }
1194             }
1195         }
1196
1197         @Override
1198         protected VrfsReactor getDataTreeChangeListener() {
1199             return VrfsReactor.this;
1200         }
1201
1202         @Override
1203         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1204             return InstanceIdentifier.create(Bgp.class).child(Vrfs.class);
1205         }
1206
1207         @Override
1208         protected synchronized void remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1209             if (ignoreClusterDcnEventForFollower()) {
1210                 return;
1211             }
1212             LOG.debug("received remove Vrfs config val {}", val.getRd());
1213             synchronized (BgpConfigurationManager.this) {
1214                 String rd = val.getRd();
1215                 BgpRouter br = getClient(YANG_OBJ);
1216                 if (br == null) {
1217                     LOG.error("{} Unable to process remove for rd {}; {}", YANG_OBJ, rd,
1218                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1219                     return;
1220                 }
1221                 try {
1222                     br.delVrf(rd);
1223                 } catch (TException | BgpRouterException e) {
1224                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1225                 }
1226             }
1227         }
1228
1229         @Override
1230         protected void update(InstanceIdentifier<Vrfs> iid,
1231                 Vrfs oldval, Vrfs newval) {
1232             if (ignoreClusterDcnEventForFollower()) {
1233                 return;
1234             }
1235             LOG.debug("VRFS: Update getting triggered for VRFS rd {}", oldval.getRd());
1236             LOG.error(YANG_OBJ + UPD_WARN);
1237         }
1238     }
1239
1240     Future lastCleanupJob;
1241     Future lastReplayJobFt = null;
1242
1243     protected void activateMIP() {
1244         try {
1245             LOG.trace("BgpReactor: Executing MIP Activate command");
1246             Process processBgp = Runtime.getRuntime().exec("cluster ip -a sdnc_bgp_mip");
1247             Process processOs = Runtime.getRuntime().exec("cluster ip -a sdnc_os_mip");
1248             LOG.trace("bgpMIP Activated");
1249
1250         } catch (IOException io) {
1251             LOG.error("IO Exception got while activating mip: {}", io.getMessage());
1252         }
1253     }
1254
1255     AtomicBoolean started = new AtomicBoolean(false);
1256
1257     public class BgpReactor
1258             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1259             implements AutoCloseable, ClusteredDataTreeChangeListener<Bgp> {
1260
1261         private static final String YANG_OBJ = "Bgp ";
1262
1263         public BgpReactor() {
1264             super(Bgp.class, BgpReactor.class);
1265         }
1266
1267
1268         @Override
1269         protected synchronized void add(InstanceIdentifier<Bgp> iid, Bgp val) {
1270             LOG.error("received add Bgp config");
1271
1272             try {
1273                 initer.await();
1274             } catch (InterruptedException e) {
1275                 // Ignored
1276             }
1277             synchronized (BgpConfigurationManager.this) {
1278                 config = val;
1279                 if (ignoreClusterDcnEventForFollower()) {
1280                     return;
1281                 }
1282                 activateMIP();
1283             }
1284         }
1285
1286         @Override
1287         protected BgpReactor getDataTreeChangeListener() {
1288             return BgpReactor.this;
1289         }
1290
1291         @Override
1292         protected InstanceIdentifier<Bgp> getWildCardPath() {
1293             return InstanceIdentifier.create(Bgp.class);
1294         }
1295
1296         @Override
1297         protected synchronized void remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1298             if (ignoreClusterDcnEventForFollower()) {
1299                 return;
1300             }
1301             LOG.debug("received remove Bgp config");
1302             synchronized (BgpConfigurationManager.this) {
1303                 config = null;
1304             }
1305         }
1306
1307         @Override
1308         protected void update(InstanceIdentifier<Bgp> iid,
1309                 Bgp oldval, Bgp newval) {
1310             if (ignoreClusterDcnEventForFollower()) {
1311                 return;
1312             }
1313             synchronized (BgpConfigurationManager.this) {
1314                 config = newval;
1315             }
1316         }
1317     }
1318
1319     @SuppressWarnings("deprecation")
1320     public class MultipathReactor
1321             extends AsyncDataTreeChangeListenerBase<Multipath, MultipathReactor>
1322             implements AutoCloseable, ClusteredDataTreeChangeListener<Multipath> {
1323
1324         private static final String YANG_OBJ = "multipath ";
1325
1326         public MultipathReactor() {
1327             super(Multipath.class, MultipathReactor.class);
1328         }
1329
1330
1331         @Override
1332         protected MultipathReactor getDataTreeChangeListener() {
1333             return MultipathReactor.this;
1334         }
1335
1336         @Override
1337         protected InstanceIdentifier<Multipath> getWildCardPath() {
1338             return InstanceIdentifier.create(Bgp.class).child(Multipath.class);
1339         }
1340
1341         @Override
1342         protected synchronized void remove(InstanceIdentifier<Multipath> iid, Multipath val) {
1343             ExecutorService executor = Executors.newSingleThreadExecutor();
1344             executor.submit(new MultipathStatusChange(val));
1345             executor.shutdown();
1346         }
1347
1348         @Override
1349         protected void update(InstanceIdentifier<Multipath> iid, Multipath oldval, Multipath newval) {
1350             ExecutorService executor = Executors.newSingleThreadExecutor();
1351             executor.submit(new MultipathStatusChange(newval));
1352             executor.shutdown();
1353         }
1354
1355         @Override
1356         protected void add(InstanceIdentifier<Multipath> key, Multipath dataObjectModification) {
1357             ExecutorService executor = Executors.newSingleThreadExecutor();
1358             executor.submit(new MultipathStatusChange(dataObjectModification));
1359             executor.shutdown();
1360         }
1361
1362         class MultipathStatusChange implements Callable<Void> {
1363
1364             Multipath multipath;
1365
1366             MultipathStatusChange(Multipath multipath) {
1367                 this.multipath = multipath;
1368             }
1369
1370             @Override
1371             public Void call() throws Exception {
1372                 if (!ignoreClusterDcnEventForFollower()) {
1373                     synchronized (BgpConfigurationManager.this) {
1374
1375                         BgpRouter br = getClient(YANG_OBJ);
1376
1377                         if (br != null) {
1378                             af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1379                             af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1380
1381                             try {
1382                                 if (multipath.isMultipathEnabled()) {
1383                                     br.enableMultipath(afi, safi);
1384                                 } else {
1385                                     br.disableMultipath(afi, safi);
1386                                 }
1387                             } catch (TException | BgpRouterException e) {
1388                                 LOG.error(YANG_OBJ + " received exception: \"" + e + "\"");
1389                             }
1390                         }
1391                     }
1392                 }
1393                 return null;
1394             }
1395
1396         }
1397
1398         @Override
1399         public void close() {
1400             super.close();
1401         }
1402     }
1403
1404     @SuppressWarnings("deprecation")
1405     public class VrfMaxpathReactor
1406             extends AsyncDataTreeChangeListenerBase<VrfMaxpath, VrfMaxpathReactor>
1407             implements AutoCloseable, ClusteredDataTreeChangeListener<VrfMaxpath> {
1408
1409         private static final String YANG_OBJ = "vrfMaxpath ";
1410
1411         public VrfMaxpathReactor() {
1412             super(VrfMaxpath.class, VrfMaxpathReactor.class);
1413         }
1414
1415
1416         @Override
1417         protected VrfMaxpathReactor getDataTreeChangeListener() {
1418             return VrfMaxpathReactor.this;
1419         }
1420
1421         @Override
1422         protected InstanceIdentifier<VrfMaxpath> getWildCardPath() {
1423             return InstanceIdentifier.create(Bgp.class).child(VrfMaxpath.class);
1424         }
1425
1426         class VrfMaxPathConfigurator implements Callable<Void> {
1427
1428             VrfMaxpath vrfMaxpathVal;
1429
1430             VrfMaxPathConfigurator(VrfMaxpath vrfMaxPathVal) {
1431                 this.vrfMaxpathVal = vrfMaxPathVal;
1432             }
1433
1434             @Override
1435             public Void call() throws Exception {
1436                 if (!ignoreClusterDcnEventForFollower()) {
1437                     synchronized (BgpConfigurationManager.this) {
1438                         BgpRouter br = getClient(YANG_OBJ);
1439                         if (br != null) {
1440                             try {
1441                                 br.multipaths(vrfMaxpathVal.getRd(), vrfMaxpathVal.getMaxpaths());
1442                                 LOG.debug("Maxpath for vrf: " + vrfMaxpathVal.getRd() + " : is "
1443                                         + vrfMaxpathVal.getMaxpaths());
1444                             } catch (TException | BgpRouterException e) {
1445                                 LOG.error(YANG_OBJ
1446                                         + " received exception: \"" + e + "\"");
1447                             }
1448                         }
1449                     }
1450                 }
1451                 return null;
1452             }
1453         }
1454
1455         @Override
1456         protected synchronized void remove(InstanceIdentifier<VrfMaxpath> iid, VrfMaxpath vrfMaxPathVal) {
1457             ExecutorService executor = Executors.newSingleThreadExecutor();
1458             executor.submit(new VrfMaxPathConfigurator(vrfMaxPathVal));
1459             executor.shutdown();
1460         }
1461
1462         @Override
1463         protected void update(InstanceIdentifier<VrfMaxpath> iid,
1464                               VrfMaxpath oldval, VrfMaxpath newval) {
1465             if (oldval.getMaxpaths() != newval.getMaxpaths()) {
1466                 ExecutorService executor = Executors.newSingleThreadExecutor();
1467                 executor.submit(new VrfMaxPathConfigurator(newval));
1468                 executor.shutdown();
1469             }
1470         }
1471
1472         @Override
1473         protected void add(InstanceIdentifier<VrfMaxpath> instanceIdentifier, VrfMaxpath vrfMaxpathVal) {
1474             ExecutorService executor = Executors.newSingleThreadExecutor();
1475             executor.submit(new VrfMaxPathConfigurator(vrfMaxpathVal));
1476             executor.shutdown();
1477         }
1478
1479         @Override
1480         public void close() {
1481             super.close();
1482         }
1483     }
1484
1485     public String readThriftIpForCommunication(String mipAddr) {
1486         File file = new File(CLUSTER_CONF_FILE);
1487         if (!file.exists()) {
1488             return DEF_CHOST;
1489         }
1490
1491         try (BufferedReader br = new BufferedReader(new FileReader(file))) {
1492             String line;
1493             while ((line = br.readLine()) != null) {
1494                 if (line.contains(mipAddr)) {
1495                     line = line.trim();
1496                     return line.substring(line.lastIndexOf(" ") + 1);
1497                 }
1498             }
1499         } catch (FileNotFoundException e) {
1500             return DEF_CHOST;
1501         } catch (IOException e) {
1502             LOG.error("Error reading {}", CLUSTER_CONF_FILE, e);
1503         }
1504         return DEF_CHOST;
1505     }
1506
1507     public boolean isIpAvailable(String odlip) {
1508
1509         try {
1510             if (odlip != null) {
1511                 if ("127.0.0.1".equals(odlip)) {
1512                     return true;
1513                 }
1514                 Enumeration networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces();
1515                 while (networkInterfaceEnumeration.hasMoreElements()) {
1516                     NetworkInterface networkInterface = (NetworkInterface) networkInterfaceEnumeration.nextElement();
1517                     Enumeration inetAddressEnumeration = networkInterface.getInetAddresses();
1518                     while (inetAddressEnumeration.hasMoreElements()) {
1519                         InetAddress inetAddress = (InetAddress) inetAddressEnumeration.nextElement();
1520                         if (odlip.equals(inetAddress.getHostAddress())) {
1521                             return true;
1522                         }
1523                     }
1524                 }
1525             }
1526         } catch (SocketException e) {
1527             // Ignored?
1528         }
1529         return false;
1530     }
1531
1532     public static long getStalePathtime(int defValue, AsId asId) {
1533         long spt = 0;
1534         try {
1535             spt = getConfig().getGracefulRestart().getStalepathTime();
1536         } catch (NullPointerException e) {
1537             try {
1538                 spt = asId.getStalepathTime();
1539                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1540             } catch (NullPointerException ignore) {
1541                 LOG.trace("BGP AS id is not set using graceful");
1542                 spt = defValue;
1543             }
1544         }
1545         if (spt == 0) {
1546             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1547             spt = defValue;
1548         }
1549         return spt;
1550     }
1551
1552     public synchronized void bgpRestarted() {
1553         /*
1554          * If there a thread which in the process of stale cleanup, cancel it
1555          * and start a new thread (to avoid processing same again).
1556          */
1557         if (previousReplayJobInProgress()) {
1558             cancelPreviousReplayJob();
1559         }
1560         Runnable task = () -> {
1561             try {
1562                 LOG.info("running bgp replay task ");
1563                 if (get() == null) {
1564                     String host = getConfigHost();
1565                     int port = getConfigPort();
1566                     LOG.info("connecting  to bgp host {} ", host);
1567
1568                     boolean res = bgpRouter.connect(host, port);
1569                     LOG.info("no config to push in bgp replay task ");
1570                     return;
1571                 }
1572                 setStaleStartTime(System.currentTimeMillis());
1573                 LOG.info("started creating stale fibDSWriter  map ");
1574                 createStaleFibMap();
1575                 setStaleEndTime(System.currentTimeMillis());
1576                 LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
1577                 LOG.info("started bgp config replay ");
1578                 setCfgReplayStartTime(System.currentTimeMillis());
1579                 try {
1580                     replay();
1581                 } catch (TimeoutException | ExecutionException e) {
1582                     LOG.error("Error while replaying routes. {}", e);
1583                 }
1584                 setCfgReplayEndTime(System.currentTimeMillis());
1585                 LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1586                 long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1587                 Thread.sleep(routeSyncTime * 1000L);
1588                 setStaleCleanupTime(routeSyncTime);
1589                 new RouteCleanup().call();
1590             } catch (InterruptedException eCancel) {
1591                 LOG.error("Stale Cleanup Task Cancelled", eCancel);
1592             }
1593         };
1594         lastReplayJobFt = executor.submit(task);
1595     }
1596
1597     private boolean previousReplayJobInProgress() {
1598         return lastReplayJobFt != null && !lastReplayJobFt.isDone();
1599     }
1600
1601     private void cancelPreviousReplayJob() {
1602         try {
1603             LOG.error("cancelling already running bgp replay task");
1604             lastReplayJobFt.cancel(true);
1605             lastReplayJobFt = null;
1606             Thread.sleep(2000);
1607         } catch (InterruptedException e) {
1608             LOG.error("Failed to cancel previous replay job ", e);
1609         }
1610     }
1611
1612     private static void doRouteSync() throws InterruptedException, TimeoutException, ExecutionException {
1613         BgpSyncHandle bsh = BgpSyncHandle.getInstance();
1614         LOG.error("Starting BGP route sync");
1615         try {
1616             bgpRouter.initRibSync(bsh);
1617         } catch (TException | BgpRouterException e) {
1618             LOG.error("Route sync aborted, exception when initializing", e);
1619             return;
1620         }
1621         while (bsh.getState() != bsh.DONE) {
1622             for (af_afi afi : af_afi.values()) {
1623                 Routes routes = null;
1624                 try {
1625                     routes = bgpRouter.doRibSync(bsh, afi);
1626                 } catch (TException | BgpRouterException e) {
1627                     LOG.error("Route sync aborted, exception when syncing", e);
1628                     return;
1629                 }
1630                 Iterator<Update> updates = routes.getUpdatesIterator();
1631                 while (updates.hasNext()) {
1632                     Update update = updates.next();
1633                     Map<String, Map<String, Long>> staleFibRdMap = BgpConfigurationManager.getStaledFibEntriesMap();
1634                     String rd = update.getRd();
1635                     String nexthop = update.getNexthop();
1636
1637                     // TODO: decide correct label here
1638                     int label = update.getL3label();
1639
1640                     String prefix = update.getPrefix();
1641                     int plen = update.getPrefixlen();
1642
1643
1644                     // TODO: protocol type will not be available in "update"
1645                     // use "rd" to query vrf table and obtain the protocol_type.
1646                     // Currently using PROTOCOL_EVPN as default.
1647                     onUpdatePushRoute(
1648                             protocol_type.PROTOCOL_EVPN,
1649                             rd,
1650                             prefix,
1651                             plen,
1652                             nexthop,
1653                             update.getEthtag(),
1654                             update.getEsi(),
1655                             update.getMacaddress(),
1656                             label,
1657                             update.getRoutermac(),
1658                             afi
1659                     );
1660                 }
1661             }
1662         }
1663         try {
1664             LOG.error("Ending BGP route-sync");
1665             bgpRouter.endRibSync(bsh);
1666         } catch (TException | BgpRouterException e) {
1667             // Ignored?
1668         }
1669     }
1670
1671     /* onUpdatePushRoute
1672      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1673      *  - Entry compare shall include NextHop, Label.
1674      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1675      *  - If entry not found, add to FIB Config DS.
1676      *  - If entry found, but either Label/NextHop doesn't match.
1677      *      - Update FIB Config DS with modified values.
1678      *      - delete from Stale Map.
1679      */
1680
1681     public static void onUpdatePushRoute(protocol_type protocolType,
1682                                          String rd,
1683                                          String prefix,
1684                                          int plen,
1685                                          String nextHop,
1686                                          int ethtag,
1687                                          String esi,
1688                                          String macaddress,
1689                                          int label,
1690                                          String routermac,
1691                                          af_afi afi)
1692             throws InterruptedException, ExecutionException, TimeoutException {
1693         boolean addroute = false;
1694         long l3vni = 0L;
1695         VrfEntry.EncapType encapType = VrfEntry.EncapType.Mplsgre;
1696         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1697             encapType = VrfEntry.EncapType.Vxlan;
1698             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = BgpUtil.getVpnInstanceOpData(dataBroker, rd);
1699             if (vpnInstanceOpDataEntry != null) {
1700                 l3vni = vpnInstanceOpDataEntry.getL3vni();
1701             } else {
1702                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1703                 return;
1704             }
1705         }
1706         if (!staledFibEntriesMap.isEmpty()) {
1707             // restart Scenario, as MAP is not empty.
1708             Map<String, Long> map = staledFibEntriesMap.get(rd);
1709             if (map != null) {
1710                 String prefixNextHop = appendNextHopToPrefix(prefix + "/" + plen, nextHop);
1711                 Long labelInStaleMap = map.get(prefixNextHop);
1712                 if (null == labelInStaleMap) {
1713                     // New Entry, which happened to be added during restart.
1714                     addroute = true;
1715                 } else {
1716                     map.remove(prefixNextHop);
1717                     if (isRouteModified(label, labelInStaleMap)) {
1718                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1719                         // Existing entry, where in Label got modified during restart
1720                         addroute = true;
1721                     }
1722                 }
1723             }
1724         } else {
1725             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1726             addroute = true;
1727         }
1728         if (addroute) {
1729             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {} afi {}",
1730                     rd, prefix, nextHop, label, afi);
1731             // TODO: modify addFibEntryToDS signature
1732             List<String> nextHopList = Collections.singletonList(nextHop);
1733             fibDSWriter.addFibEntryToDS(rd, macaddress, prefix + "/" + plen, nextHopList, encapType, label, l3vni,
1734                                         routermac, RouteOrigin.BGP);
1735             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1736             String vpnName = BgpUtil.getVpnNameFromRd(dataBroker, rd);
1737             if (vpnName != null) {
1738                 vpnLinkService.leakRouteIfNeeded(vpnName, prefix, nextHopList, label, RouteOrigin.BGP,
1739                                                  NwConstants.ADD_FLOW);
1740             }
1741         }
1742     }
1743
1744     private static boolean isRouteModified(int label, Long labelInStaleMap) {
1745         return labelInStaleMap != null && !labelInStaleMap.equals(Long.valueOf(label));
1746     }
1747
1748     private static void replayNbrConfig(List<Neighbors> neighbors, BgpRouter br) {
1749         for (Neighbors nbr : neighbors) {
1750             try {
1751                 final String md5password = extractMd5Secret(nbr);
1752                 br.addNeighbor(nbr.getAddress().getValue(),
1753                         nbr.getRemoteAs(), md5password);
1754                 //itmProvider.buildTunnelsToDCGW(new IpAddress(nbr.getAddress().getValue().toCharArray()));
1755             } catch (TException | BgpRouterException e) {
1756                 LOG.error("Replay:addNbr() received exception", e);
1757                 continue;
1758             }
1759             EbgpMultihop en = nbr.getEbgpMultihop();
1760             if (en != null) {
1761                 try {
1762                     br.addEbgpMultihop(en.getPeerIp().getValue(),
1763                             en.getNhops().intValue());
1764                 } catch (TException | BgpRouterException e) {
1765                     LOG.error("Replay:addEBgp() received exception", e);
1766                 }
1767             }
1768             UpdateSource us = nbr.getUpdateSource();
1769             if (us != null) {
1770                 try {
1771                     br.addUpdateSource(us.getPeerIp().getValue(),
1772                             us.getSourceIp().getValue());
1773                 } catch (TException | BgpRouterException e) {
1774                     LOG.error("Replay:addUS() received exception", e);
1775                 }
1776             }
1777             List<AddressFamilies> afs = nbr.getAddressFamilies();
1778             if (afs != null) {
1779                 for (AddressFamilies af : afs) {
1780                     af_afi afi = af_afi.findByValue(af.getAfi().intValue());
1781                     af_safi safi = af_safi.findByValue(af.getSafi().intValue());
1782                     try {
1783                         br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
1784                     } catch (TException | BgpRouterException e) {
1785                         LOG.error("Replay:addAf() received exception", e);
1786                     }
1787                 }
1788             }
1789         }
1790     }
1791
1792     public static String getConfigHost() {
1793         if (config == null) {
1794             return cHostStartup;
1795         }
1796         ConfigServer ts = config.getConfigServer();
1797         return (ts == null ? cHostStartup : ts.getHost().getValue());
1798     }
1799
1800     public static int getConfigPort() {
1801         if (config == null) {
1802             return Integer.parseInt(cPortStartup);
1803         }
1804         ConfigServer ts = config.getConfigServer();
1805         return (ts == null ? Integer.parseInt(cPortStartup) :
1806                 ts.getPort().intValue());
1807     }
1808
1809     public static Bgp getConfig() {
1810         AtomicInteger bgpDSretryCount = new AtomicInteger(DS_RETRY_COOUNT);
1811         while (0 != bgpDSretryCount.decrementAndGet()) {
1812             try {
1813                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
1814                         InstanceIdentifier.create(Bgp.class)).orNull();
1815             } catch (ReadFailedException e) {
1816                 //Config DS may not be up, so sleep for 1 second and retry
1817                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
1818                 try {
1819                     Thread.sleep(WAIT_TIME_BETWEEN_EACH_TRY_MILLIS);
1820                 } catch (InterruptedException timerEx) {
1821                     LOG.debug("WAIT_TIME_BETWEEN_EACH_TRY_MILLIS, Timer got interrupted while waiting for"
1822                             + "config DS availability", timerEx);
1823                 }
1824             }
1825         }
1826         LOG.error("failed to get bgp config");
1827         return null;
1828     }
1829
1830     @SuppressWarnings("checkstyle:IllegalCatch")
1831     public synchronized void replay() throws InterruptedException, TimeoutException, ExecutionException {
1832         synchronized (bgpConfigurationManager) {
1833             String host = getConfigHost();
1834             int port = getConfigPort();
1835             LOG.error("connecting  to bgp host {} ", host);
1836
1837             boolean res = bgpRouter.connect(host, port);
1838             if (!res) {
1839                 String msg = "Cannot connect to BGP config server at " + host + ":" + port;
1840                 if (config != null) {
1841                     msg += "; Configuration Replay aborted";
1842                 }
1843                 LOG.error(msg);
1844                 return;
1845             }
1846             config = getConfig();
1847             if (config == null) {
1848                 LOG.error("bgp config is empty nothing to push to bgp");
1849                 return;
1850             }
1851             BgpRouter br = bgpRouter;
1852             AsId asId = config.getAsId();
1853             if (asId == null) {
1854                 return;
1855             }
1856             long asNum = asId.getLocalAs();
1857             IpAddress routerId = asId.getRouterId();
1858             Long spt = asId.getStalepathTime();
1859             Boolean afb = asId.isAnnounceFbit();
1860             String rid = (routerId == null) ? "" : new String(routerId.getValue());
1861             int stalepathTime = (int) getStalePathtime(RESTART_DEFAULT_GR, config.getAsId());
1862             boolean announceFbit = true;
1863             try {
1864                 br.startBgp(asNum, rid, stalepathTime, announceFbit);
1865             } catch (BgpRouterException bre) {
1866                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
1867                     doRouteSync();
1868                 } else {
1869                     LOG.error("Replay: startBgp() received exception: \""
1870                             + bre + "\"; " + ADD_WARN);
1871                 }
1872             } catch (TException e) {
1873                 //not unusual. We may have restarted & BGP is already on
1874                 LOG.error("Replay:startBgp() received exception: \"" + e + "\"");
1875             }
1876
1877             if (getBgpCounters() == null) {
1878                 startBgpCountersTask();
1879             }
1880
1881             if (getBgpAlarms() == null) {
1882                 startBgpAlarmsTask();
1883             }
1884
1885             Logging logging = config.getLogging();
1886             if (logging != null) {
1887                 try {
1888                     br.setLogging(logging.getFile(), logging.getLevel());
1889                 } catch (TException | BgpRouterException e) {
1890                     LOG.error("Replay:setLogging() received exception", e);
1891                 }
1892             }
1893
1894             GracefulRestart gracefulRestart = config.getGracefulRestart();
1895             if (gracefulRestart != null) {
1896                 try {
1897                     br.addGracefulRestart(gracefulRestart.getStalepathTime().intValue());
1898                 } catch (TException | BgpRouterException e) {
1899                     LOG.error("Replay:addGr() received exception", e);
1900                 }
1901             }
1902
1903             List<Vrfs> vrfs = config.getVrfs();
1904             if (vrfs != null) {
1905                 for (Vrfs vrf : vrfs) {
1906                     try {
1907                         br.addVrf(vrf.getLayerType(), vrf.getRd(), vrf.getImportRts(),
1908                                 vrf.getExportRts());
1909                     } catch (TException | BgpRouterException e) {
1910                         LOG.error("Replay:addVrf() received exception", e);
1911                     }
1912                 }
1913             }
1914
1915             List<Networks> ln = config.getNetworks();
1916             if (ln != null) {
1917                 for (Networks net : ln) {
1918                     String rd = net.getRd();
1919                     String pfxlen = net.getPrefixLen();
1920                     String nh = net.getNexthop().getValue();
1921                     Long label = net.getLabel();
1922                     int lbl = (label == null) ? 0 : label.intValue();
1923                     int l3vni = (net.getL3vni() == null) ? 0 : net.getL3vni().intValue();
1924                     Long afi = net.getAfi();
1925                     int afint = (afi == null) ? (int) af_afi.AFI_IP.getValue() : afi.intValue();
1926                     if (rd == null && lbl > 0) {
1927                         //LU prefix is being deleted.
1928                         rd = Integer.toString(lbl);
1929                     }
1930
1931                     BgpControlPlaneType protocolType = net.getBgpControlPlaneType();
1932                     int ethernetTag = net.getEthtag().intValue();
1933                     String esi = net.getEsi();
1934                     String macaddress = net.getMacaddress();
1935                     EncapType encapType = net.getEncapType();
1936                     String routerMac = net.getRoutermac();
1937
1938                     try {
1939                         br.addPrefix(rd, pfxlen, nh, lbl, l3vni, BgpUtil.convertToThriftProtocolType(protocolType),
1940                                 ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType),
1941                                 routerMac);
1942                     } catch (Exception e) {
1943                         LOG.error("Replay:addPfx() received exception", e);
1944                     }
1945                 }
1946             }
1947             List<Neighbors> neighbors = config.getNeighbors();
1948             if (neighbors != null) {
1949                 LOG.error("configuring existing Neighbors present for replay total neighbors {}", neighbors.size());
1950                 replayNbrConfig(neighbors, br);
1951             } else {
1952                 LOG.error("no Neighbors present for replay config ");
1953             }
1954
1955             List<Multipath> multipaths = config.getMultipath();
1956
1957             if (multipaths != null) {
1958                 for (Multipath multipath: multipaths) {
1959                     if (multipath != null) {
1960                         af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1961                         af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1962
1963                         try {
1964                             if (multipath.isMultipathEnabled()) {
1965                                 br.enableMultipath(afi, safi);
1966                             } else {
1967                                 br.disableMultipath(afi, safi);
1968                             }
1969                         } catch (TException | BgpRouterException e) {
1970                             LOG.info("Replay:multipaths() received exception: \"" + e + "\"");
1971                         }
1972                     }
1973                 }
1974             }
1975             List<VrfMaxpath> vrfMaxpaths = config.getVrfMaxpath();
1976             if (vrfMaxpaths != null) {
1977                 for (VrfMaxpath vrfMaxpath: vrfMaxpaths) {
1978                     try {
1979                         br.multipaths(vrfMaxpath.getRd(), vrfMaxpath.getMaxpaths());
1980                     } catch (TException | BgpRouterException e) {
1981                         LOG.info("Replay:vrfMaxPath() received exception: \"" + e + "\"");
1982                     }
1983                 }
1984             }
1985         }
1986     }
1987
1988     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
1989         BgpUtil.update(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1990     }
1991
1992     private <T extends DataObject> void asyncWrite(InstanceIdentifier<T> iid, T dto) {
1993         BgpUtil.write(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1994     }
1995
1996     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
1997         BgpUtil.delete(dataBroker, LogicalDatastoreType.CONFIGURATION, iid);
1998     }
1999
2000     public void startConfig(String bgpHost, int thriftPort) {
2001         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2002                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2003         InstanceIdentifier<ConfigServer> iid = iib.build();
2004         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
2005         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
2006                 .setPort((long) thriftPort).build();
2007         update(iid, dto);
2008     }
2009
2010     public void startBgp(long as, String routerId, int spt, boolean fbit) {
2011         IpAddress rid = (routerId == null) ? null : new IpAddress(routerId.toCharArray());
2012         Long staleTime = (long) spt;
2013         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2014                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2015         InstanceIdentifier<AsId> iid = iib.build();
2016         AsId dto = new AsIdBuilder().setLocalAs(as)
2017                 .setRouterId(rid)
2018                 .setStalepathTime(staleTime)
2019                 .setAnnounceFbit(fbit).build();
2020         update(iid, dto);
2021     }
2022
2023     public void addLogging(String fileName, String logLevel) {
2024         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2025                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2026         InstanceIdentifier<Logging> iid = iib.build();
2027         Logging dto = new LoggingBuilder().setFile(fileName)
2028                 .setLevel(logLevel).build();
2029         update(iid, dto);
2030     }
2031
2032     public void addGracefulRestart(int staleTime) {
2033         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2034                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
2035         InstanceIdentifier<GracefulRestart> iid = iib.build();
2036         GracefulRestart dto = new GracefulRestartBuilder()
2037                 .setStalepathTime((long) staleTime).build();
2038         update(iid, dto);
2039     }
2040
2041     public void addNeighbor(
2042             String nbrIp, long remoteAs, @Nullable final TcpMd5SignaturePasswordType md5Secret) {
2043         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2044         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2045                 InstanceIdentifier.builder(Bgp.class)
2046                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2047         InstanceIdentifier<Neighbors> iid = iib.build();
2048         TcpSecurityOption tcpSecOption = null;
2049         if (md5Secret != null) {
2050             tcpSecOption = new TcpMd5SignatureOptionBuilder().setTcpMd5SignaturePassword(md5Secret).build();
2051         } // else let tcpSecOption be null
2052         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
2053                 .setRemoteAs(remoteAs).setTcpSecurityOption(tcpSecOption).build();
2054         update(iid, dto);
2055     } // public addNeighbor(nbrIp, remoteAs, md5Secret)
2056
2057     public void addUpdateSource(String nbrIp, String srcIp) {
2058         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2059         Ipv4Address srcAddr = new Ipv4Address(srcIp);
2060         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2061                 InstanceIdentifier.builder(Bgp.class)
2062                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2063                         .child(UpdateSource.class);
2064         InstanceIdentifier<UpdateSource> iid = iib.build();
2065         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
2066                 .setSourceIp(srcAddr).build();
2067         update(iid, dto);
2068     }
2069
2070     public void addEbgpMultihop(String nbrIp, int hops) {
2071         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2072         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2073                 InstanceIdentifier.builder(Bgp.class)
2074                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2075                         .child(EbgpMultihop.class);
2076         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2077         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
2078                 .setNhops((long) hops).build();
2079         update(iid, dto);
2080     }
2081
2082     public void addAddressFamily(String nbrIp, int afi, int safi) {
2083         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2084         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2085                 InstanceIdentifier.builder(Bgp.class)
2086                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2087                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2088         InstanceIdentifier<AddressFamilies> iid = iib.build();
2089         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
2090                 .setAfi((long) afi).setSafi((long) safi).build();
2091         update(iid, dto);
2092     }
2093
2094     public void addPrefix(String rd, String macAddress, String pfx, List<String> nhList,
2095               VrfEntry.EncapType encapType, long lbl, long l3vni, long l2vni, String gatewayMac) {
2096         for (String nh : nhList) {
2097             Ipv4Address nexthop = nh != null ? new Ipv4Address(nh) : null;
2098             Long label = lbl;
2099             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
2100                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
2101             NetworksBuilder networksBuilder = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
2102                                                 .setLabel(label).setEthtag(BgpConstants.DEFAULT_ETH_TAG);
2103             buildVpnEncapSpecificInfo(networksBuilder, encapType, label, l3vni, l2vni, macAddress, gatewayMac);
2104             update(iid, networksBuilder.build());
2105         }
2106     }
2107
2108     private static void buildVpnEncapSpecificInfo(NetworksBuilder builder, VrfEntry.EncapType encapType, long label,
2109                                                   long l3vni, long l2vni, String macAddress, String gatewayMac) {
2110         if (encapType.equals(VrfEntry.EncapType.Mplsgre)) {
2111             builder.setLabel(label).setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLL3VPN)
2112                     .setEncapType(EncapType.GRE);
2113         } else {
2114             builder.setL3vni(l3vni).setL2vni(l2vni).setMacaddress(macAddress).setRoutermac(gatewayMac)
2115                     .setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLEVPN).setEncapType(EncapType.VXLAN);
2116         }
2117     }
2118
2119     // TODO: add LayerType as arg - supports command
2120     public void addVrf(String rd, List<String> irts, List<String> erts, LayerType layerType) {
2121         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2122                 InstanceIdentifier.builder(Bgp.class)
2123                         .child(Vrfs.class, new VrfsKey(rd));
2124         InstanceIdentifier<Vrfs> iid = iib.build();
2125         Vrfs dto = new VrfsBuilder().setRd(rd).setImportRts(irts)
2126                 .setExportRts(erts).setLayerType(layerType).build();
2127         try {
2128             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
2129         } catch (TransactionCommitFailedException e) {
2130             LOG.error("Error adding VRF to datastore", e);
2131             throw new RuntimeException(e);
2132         }
2133     }
2134
2135     public void stopConfig() {
2136         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2137                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2138         InstanceIdentifier<ConfigServer> iid = iib.build();
2139         delete(iid);
2140     }
2141
2142     public void stopBgp() {
2143         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2144                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2145         InstanceIdentifier<AsId> iid = iib.build();
2146         delete(iid);
2147     }
2148
2149     public void delLogging() {
2150         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2151                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2152         InstanceIdentifier<Logging> iid = iib.build();
2153         delete(iid);
2154     }
2155
2156     public void delGracefulRestart() {
2157         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2158                 InstanceIdentifier.builder(Bgp.class)
2159                         .child(GracefulRestart.class);
2160         InstanceIdentifier<GracefulRestart> iid = iib.build();
2161         delete(iid);
2162     }
2163
2164     public void delNeighbor(String nbrIp) {
2165         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2166         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2167                 InstanceIdentifier.builder(Bgp.class)
2168                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2169         InstanceIdentifier<Neighbors> iid = iib.build();
2170         delete(iid);
2171     }
2172
2173     public void delUpdateSource(String nbrIp) {
2174         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2175         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2176                 InstanceIdentifier.builder(Bgp.class)
2177                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2178                         .child(UpdateSource.class);
2179         InstanceIdentifier<UpdateSource> iid = iib.build();
2180         delete(iid);
2181     }
2182
2183     public void delEbgpMultihop(String nbrIp) {
2184         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2185         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2186                 InstanceIdentifier.builder(Bgp.class)
2187                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2188                         .child(EbgpMultihop.class);
2189         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2190         delete(iid);
2191     }
2192
2193     public void delAddressFamily(String nbrIp, int afi, int safi) {
2194         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2195         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2196                 InstanceIdentifier.builder(Bgp.class)
2197                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2198                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2199         InstanceIdentifier<AddressFamilies> iid = iib.build();
2200         delete(iid);
2201     }
2202
2203     public void delPrefix(String rd, String pfx) {
2204         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
2205                 InstanceIdentifier.builder(Bgp.class)
2206                         .child(Networks.class, new NetworksKey(pfx, rd));
2207         InstanceIdentifier<Networks> iid = iib.build();
2208         delete(iid);
2209     }
2210
2211     public void delVrf(String rd) {
2212         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2213                 InstanceIdentifier.builder(Bgp.class)
2214                         .child(Vrfs.class, new VrfsKey(rd));
2215         InstanceIdentifier<Vrfs> iid = iib.build();
2216         delete(iid);
2217     }
2218
2219     public void setMultipathStatus(af_afi afi, af_safi safi, boolean enable) {
2220         long lafi = afi.getValue();
2221         long lsafi = safi.getValue();
2222
2223         InstanceIdentifier.InstanceIdentifierBuilder<Multipath> iib =
2224                 InstanceIdentifier
2225                         .builder(Bgp.class)
2226                         .child(Multipath.class,
2227                                 new MultipathKey(Long.valueOf(afi.getValue()), Long.valueOf(safi.getValue())));
2228
2229         Multipath dto = new MultipathBuilder().setAfi(lafi).setSafi(lsafi).setMultipathEnabled(enable).build();
2230         update(iib.build(), dto);
2231     }
2232
2233     public void multipaths(String rd, int maxpath) {
2234         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
2235                 InstanceIdentifier
2236                         .builder(Bgp.class)
2237                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
2238
2239         VrfMaxpath dto = new VrfMaxpathBuilder().setRd(rd).setMaxpaths(maxpath).build();
2240         update(iib.build(), dto);
2241     }
2242
2243     static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
2244
2245     /*
2246     * Remove Stale Marked Routes after timer expiry.
2247     */
2248     class RouteCleanup implements Callable<Integer> {
2249
2250         @Override
2251         public Integer call() {
2252             totalCleared = 0;
2253             try {
2254                 if (staledFibEntriesMap.isEmpty()) {
2255                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
2256                 } else {
2257                     for (String rd : staledFibEntriesMap.keySet()) {
2258                         if (Thread.interrupted()) {
2259                             return 0;
2260                         }
2261                         Map<String, Long> map = staledFibEntriesMap.get(rd);
2262                         if (map != null) {
2263                             for (String key : map.keySet()) {
2264                                 if (Thread.interrupted()) {
2265                                     return 0;
2266                                 }
2267                                 String prefix = extractPrefix(key);
2268                                 String nextHop = extractNextHop(key);
2269                                 totalCleared++;
2270                                 LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}, nextHop:{}",
2271                                         rd, prefix, nextHop);
2272                                 fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix, nextHop);
2273                             }
2274                         }
2275                     }
2276                 }
2277             } finally {
2278                 staledFibEntriesMap.clear();
2279             }
2280             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
2281             return 0;
2282         }
2283     }
2284
2285     /*
2286      * BGP restart scenario, ODL-BGP manager was/is running.
2287      * On re-sync notification, Get a copy of FIB database.
2288      */
2289     public static void createStaleFibMap() {
2290         totalStaledCount = 0;
2291         try {
2292             /*
2293             * at the time Stale FIB creation, Wait till all PENDING write transaction
2294              * to complete (or)wait for max timeout value of STALE_FIB_WAIT Seconds.
2295              */
2296             int retry = STALE_FIB_WAIT;
2297             while ((BgpUtil.getGetPendingWrTransaction() != 0) && (retry > 0)) {
2298                 Thread.sleep(1000);
2299                 retry--;
2300                 if (retry == 0) {
2301                     LOG.error("TimeOut occured {} seconds, in waiting stale fibDSWriter create", STALE_FIB_WAIT);
2302                 }
2303             }
2304             staledFibEntriesMap.clear();
2305             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2306             DataBroker db = BgpUtil.getBroker();
2307             if (db == null) {
2308                 LOG.error("Couldn't find BgpUtil dataBroker while creating createStaleFibMap");
2309                 return;
2310             }
2311
2312             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(BgpUtil.getBroker(),
2313                     LogicalDatastoreType.CONFIGURATION, id);
2314             if (fibEntries.isPresent()) {
2315                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2316                 for (VrfTables vrfTable : staleVrfTables) {
2317                     Map<String, Long> staleFibEntMap = new HashMap<>();
2318                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2319                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2320                             //Stale marking and cleanup is only meant for the routes learned through BGP.
2321                             continue;
2322                         }
2323                         if (Thread.interrupted()) {
2324                             break;
2325                         }
2326                         totalStaledCount++;
2327                         //Create MAP from staleVrfTables.
2328                         vrfEntry.getRoutePaths()
2329                                 .forEach(
2330                                     routePath -> staleFibEntMap.put(
2331                                             appendNextHopToPrefix(vrfEntry.getDestPrefix(),
2332                                                     routePath.getNexthopAddress()), routePath.getLabel()));
2333                     }
2334                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), staleFibEntMap);
2335                 }
2336             } else {
2337                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
2338             }
2339         } catch (InterruptedException | ReadFailedException e) {
2340             LOG.error("createStaleFibMap:: error ", e);
2341         }
2342         LOG.error("created {} staled entries ", totalStaledCount);
2343     }
2344
2345     /*
2346      * BGP config remove scenario, Need to remove all the
2347      * external routes from FIB.
2348      */
2349     public static void deleteExternalFibRoutes() {
2350         totalExternalRoutes = 0;
2351         try {
2352             /*
2353             * at the time FIB route deletion, Wait till all PENDING write transaction
2354              * to complete (or)wait for max timeout value of STALE_FIB_WAIT Seconds.
2355              */
2356             int retry = STALE_FIB_WAIT;
2357             String rd;
2358             while ((BgpUtil.getGetPendingWrTransaction() != 0) && (retry > 0)) {
2359                 Thread.sleep(1000);
2360                 retry--;
2361                 if (retry == 0) {
2362                     LOG.error("TimeOut occured {} seconds, while deleting external routes", STALE_FIB_WAIT);
2363                 }
2364             }
2365             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2366             DataBroker db = BgpUtil.getBroker();
2367             if (db == null) {
2368                 LOG.error("Couldn't find BgpUtil dataBroker while deleting external routes");
2369                 return;
2370             }
2371
2372             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(BgpUtil.getBroker(),
2373                     LogicalDatastoreType.CONFIGURATION, id);
2374             if (fibEntries.isPresent()) {
2375                 if (fibEntries.get().getVrfTables() == null) {
2376                     LOG.error("deleteExternalFibRoutes::getVrfTables is null");
2377                     return;
2378                 }
2379                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2380                 for (VrfTables vrfTable : staleVrfTables) {
2381                     rd = vrfTable.getRouteDistinguisher();
2382                     if (vrfTable.getVrfEntry() == null) {
2383                         LOG.error("deleteExternalFibRoutes::getVrfEntry is null");
2384                         continue;
2385                     }
2386                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2387                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2388                             //route cleanup is only meant for the routes learned through BGP.
2389                             continue;
2390                         }
2391                         totalExternalRoutes++;
2392                         fibDSWriter.removeFibEntryFromDS(rd, vrfEntry.getDestPrefix());
2393                     }
2394                 }
2395             } else {
2396                 LOG.error("deleteExternalFibRoutes:: FIBentries.class is not present");
2397             }
2398         } catch (InterruptedException | ReadFailedException e) {
2399             LOG.error("deleteExternalFibRoutes:: error ", e);
2400         }
2401         LOG.debug("deleted {} fib entries ", totalExternalRoutes);
2402     }
2403
2404     //map<rd, map<prefix/len:nexthop, label>>
2405     public static Map<String, Map<String, Long>> getStaledFibEntriesMap() {
2406         return staledFibEntriesMap;
2407     }
2408
2409     //TODO: below function is for testing purpose with cli
2410     public static void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
2411         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
2412         fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nexthop);
2413         String vpnName = BgpUtil.getVpnNameFromRd(dataBroker, rd);
2414         if (vpnName != null) {
2415             vpnLinkService.leakRouteIfNeeded(vpnName, prefix, null /*nextHopList*/, 0 /*INVALID_LABEL*/,
2416                                              RouteOrigin.BGP, NwConstants.DEL_FLOW);
2417         }
2418     }
2419
2420     public boolean isBgpConnected() {
2421         return bgpRouter.isBgpConnected();
2422     }
2423
2424     public long getLastConnectedTS() {
2425         return bgpRouter.getLastConnectedTS();
2426     }
2427
2428     public long getConnectTS() {
2429         return bgpRouter.getConnectTS();
2430     }
2431
2432     public long getStartTS() {
2433         return bgpRouter.getStartTS();
2434     }
2435
2436     public static int getTotalStaledCount() {
2437         return totalStaledCount;
2438     }
2439
2440     public static int getTotalCleared() {
2441         return totalCleared;
2442     }
2443
2444     public Timer getBgpCountersTimer() {
2445         return bgpCountersTimer;
2446     }
2447
2448     public BgpCounters getBgpCounters() {
2449         return bgpCounters;
2450     }
2451
2452     public void setBgpCountersTimer(Timer timer) {
2453         bgpCountersTimer = timer;
2454     }
2455
2456     public void setBgpAlarmsTimer(Timer timer) {
2457         bgpAlarmsTimer = timer;
2458     }
2459
2460     public void startBgpCountersTask() {
2461         if (getBgpCounters() == null) {
2462             bgpCounters = new BgpCounters(bgpConfigurationManager.getBgpSdncMipIp());
2463             setBgpCountersTimer(new Timer(true));
2464             getBgpCountersTimer().scheduleAtFixedRate(bgpCounters, 0, 120 * 1000);
2465             LOG.info("Bgp Counters task scheduled for every two minutes.");
2466
2467             bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
2468         }
2469     }
2470
2471     public void stopBgpCountersTask() {
2472         Timer timer = getBgpCountersTimer();
2473         if (getBgpCounters() != null) {
2474             timer.cancel();
2475             setBgpCountersTimer(null);
2476             bgpCounters = null;
2477         }
2478     }
2479
2480     public void startBgpAlarmsTask() {
2481         if (getBgpAlarms() == null) {
2482             bgpAlarms = new BgpAlarms(this);
2483             setBgpAlarmsTimer(new Timer(true));
2484             getBgpAlarmsTimer().scheduleAtFixedRate(bgpAlarms, 0, 60 * 1000);
2485             LOG.info("Bgp Alarms task scheduled for every minute.");
2486         }
2487     }
2488
2489     public void stopBgpAlarmsTask() {
2490         Timer timer = getBgpAlarmsTimer();
2491         if (getBgpAlarms() != null) {
2492             timer.cancel();
2493             setBgpAlarmsTimer(null);
2494             bgpAlarms = null;
2495         }
2496     }
2497
2498     public Timer getBgpAlarmsTimer() {
2499         return bgpAlarmsTimer;
2500     }
2501
2502     public BgpAlarms getBgpAlarms() {
2503         return bgpAlarms;
2504     }
2505
2506     private static String appendNextHopToPrefix(String prefix, String nextHop) {
2507         return prefix + ":" + nextHop;
2508     }
2509
2510     private static String extractPrefix(String prefixNextHop) {
2511         return prefixNextHop.split(":")[0];
2512     }
2513
2514     private static String extractNextHop(String prefixNextHop) {
2515         return prefixNextHop.split(":")[1];
2516     }
2517
2518     private static String extractMd5Secret(final Neighbors val) {
2519         String md5Secret = null;
2520         TcpSecurityOption tcpSecOpt = val.getTcpSecurityOption();
2521         if (tcpSecOpt != null) {
2522             if (tcpSecOpt instanceof TcpMd5SignatureOption) {
2523                 md5Secret = ((TcpMd5SignatureOption) tcpSecOpt).getTcpMd5SignaturePassword().getValue();
2524             } else { // unknown TcpSecurityOption
2525                 LOG.debug("neighbors  Ignored unknown tcp-security-option of peer {}", val.getAddress().getValue());
2526             }
2527         }
2528         return md5Secret;
2529     } // private method extractMd5Secret
2530 }