vpnservice/bgpmanager/bgpmanager-impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import com.google.common.base.Optional;
  11 import io.netty.util.concurrent.GlobalEventExecutor;
  12 import java.io.BufferedReader;
  13 import java.io.File;
  14 import java.io.FileNotFoundException;
  15 import java.io.FileReader;
  16 import java.io.IOException;
  17 import java.lang.reflect.Constructor;
  18 import java.lang.reflect.InvocationTargetException;
  19 import java.net.InetAddress;
  20 import java.net.NetworkInterface;
  21 import java.net.SocketException;
  22 import java.util.Collections;
  23 import java.util.Enumeration;
  24 import java.util.HashMap;
  25 import java.util.Iterator;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Timer;
  29 import java.util.TimerTask;
  30 import java.util.concurrent.Callable;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.CountDownLatch;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.concurrent.ExecutorService;
  35 import java.util.concurrent.Executors;
  36 import java.util.concurrent.Future;
  37 import java.util.concurrent.ScheduledExecutorService;
  38 import java.util.concurrent.TimeoutException;
  39 import java.util.concurrent.atomic.AtomicBoolean;
  40 import java.util.concurrent.atomic.AtomicInteger;
  41 import javax.annotation.Nullable;
  42
  43
  44 import org.apache.thrift.TException;
  45 import org.opendaylight.controller.config.api.osgi.WaitingServiceTracker;
  46 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  47 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  48 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  49 import org.opendaylight.controller.md.sal.common.api.clustering.CandidateAlreadyRegisteredException;
  50 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipService;
  51 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  52 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  53 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  54 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  55 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  56 import org.opendaylight.genius.utils.batching.DefaultBatchHandler;
  57 import org.opendaylight.genius.utils.clustering.EntityOwnerUtils;
  58 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.AddressFamiliesReactor;
  59 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.AsIdReactor;
  60 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.BgpReactor;
  61 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.ConfigServerReactor;
  62 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.EbgpMultihopReactor;
  63 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.GracefulRestartReactor;
  64 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.LoggingReactor;
  65 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.MultipathReactor;
  66 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.NeighborsReactor;
  67 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.NetworksReactor;
  68 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.RouteCleanup;
  69 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.UpdateSourceReactor;
  70 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.VrfMaxpathReactor;
  71 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.VrfsReactor;
  72 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  73 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  74 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  75 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  76 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  77 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  78 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  79 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  80 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  81 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  82 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  83 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  84 import org.opendaylight.netvirt.bgpmanager.thrift.gen.protocol_type;
  85 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  86 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  87 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  88 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  89 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
  90 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EncapType;
  91 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.LayerType;
  92 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.TcpMd5SignaturePasswordType;
  93 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
  94 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
  96 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
  98 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
 100 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Multipath;
 102 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathKey;
 104 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Neighbors;
 105 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsKey;
 107 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Networks;
 108 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksKey;
 110 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpath;
 111 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathKey;
 113 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Vrfs;
 114 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsKey;
 116 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamilies;
 117 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesBuilder;
 118 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesKey;
 119 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihop;
 120 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihopBuilder;
 121 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSource;
 122 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSourceBuilder;
 123 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.TcpSecurityOption;
 124 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOptionBuilder;
 125 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
 126 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 131 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 132 import org.opendaylight.yangtools.yang.binding.DataObject;
 133 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 134 import org.osgi.framework.BundleContext;
 135 import org.slf4j.Logger;
 136 import org.slf4j.LoggerFactory;
 137
 138
 139
 140 public class BgpConfigurationManager {
 141     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 142     private static DataBroker dataBroker;
 143     private static FibDSWriter fibDSWriter;
 144     public static IBgpManager bgpManager;
 145     private final BundleContext bundleContext;
 146     private static Bgp config;
 147     private static BgpRouter bgpRouter;
 148     private static BgpThriftService updateServer;
 149     private BgpCounters bgpCounters;
 150     private BgpAlarms bgpAlarms;
 151     private Timer bgpCountersTimer;
 152     private Timer bgpAlarmsTimer;
 153     private static final String DEF_LOGFILE = "/var/log/bgp_debug.log";
 154     private static final String DEF_LOGLEVEL = "errors";
 155     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 156     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 157     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 158     private static final String DEF_UPORT = "6644";
 159     private static final String DEF_CHOST = "127.0.0.1";
 160     private static final String DEF_CPORT = "7644";
 161     private static final String SDNC_BGP_MIP = "sdnc_bgp_mip";
 162     private static final String BGP_SDNC_MIP = "bgp_sdnc_mip";
 163     private static final String CLUSTER_CONF_FILE = "/cluster/etc/cluster.conf";
 164     private static final Timer IP_ACTIVATION_CHECK_TIMER = new Timer();
 165     private static final int STALE_FIB_WAIT = 60;
 166     private static final int RESTART_DEFAULT_GR = 90;
 167     private long staleStartTime = 0;
 168     private long staleEndTime = 0;
 169     private long cfgReplayStartTime = 0;
 170     private long cfgReplayEndTime = 0;
 171     private long staleCleanupTime = 0;
 172     private static final int DS_RETRY_COOUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
 173     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
 174
 175     public String getBgpSdncMipIp() {
 176         return readThriftIpForCommunication(BGP_SDNC_MIP);
 177     }
 178
 179     public long getStaleCleanupTime() {
 180         return staleCleanupTime;
 181     }
 182
 183     public void setStaleCleanupTime(long staleCleanupTime) {
 184         this.staleCleanupTime = staleCleanupTime;
 185     }
 186
 187     public long getCfgReplayEndTime() {
 188         return cfgReplayEndTime;
 189     }
 190
 191     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 192         this.cfgReplayEndTime = cfgReplayEndTime;
 193     }
 194
 195     public long getCfgReplayStartTime() {
 196         return cfgReplayStartTime;
 197     }
 198
 199     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 200         this.cfgReplayStartTime = cfgReplayStartTime;
 201     }
 202
 203     public long getStaleEndTime() {
 204         return staleEndTime;
 205     }
 206
 207     public void setStaleEndTime(long staleEndTime) {
 208         this.staleEndTime = staleEndTime;
 209     }
 210
 211     public long getStaleStartTime() {
 212         return staleStartTime;
 213     }
 214
 215     public void setStaleStartTime(long staleStartTime) {
 216         this.staleStartTime = staleStartTime;
 217     }
 218
 219
 220     // to have stale FIB map (RD, Prefix)
 221     //  number of seconds wait for route sync-up between ODL and BGP
 222     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 223
 224     static String odlThriftIp = "127.0.0.1";
 225     static String bgpThriftIp = "127.0.0.1";
 226     private static String cHostStartup;
 227     private static String cPortStartup;
 228     private static CountDownLatch initer = new CountDownLatch(1);
 229     //static IITMProvider itmProvider;
 230     //map<rd, map<prefix/len:nexthop, label>>
 231     private static Map<String, Map<String, Long>> staledFibEntriesMap = new ConcurrentHashMap<>();
 232
 233     static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 234     static final String BGP_ENTITY_NAME = "bgp";
 235
 236     static int totalStaledCount = 0;
 237     static int totalCleared = 0;
 238
 239     private static final Class[] REACTORS = {
 240         ConfigServerReactor.class, AsIdReactor.class,
 241         GracefulRestartReactor.class, LoggingReactor.class,
 242         NeighborsReactor.class, UpdateSourceReactor.class,
 243         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 244         NetworksReactor.class, VrfsReactor.class, BgpReactor.class,
 245         MultipathReactor.class, VrfMaxpathReactor.class
 246     };
 247
 248     private ListenerRegistration<DataChangeListener>[] registrations;
 249
 250     final BgpConfigurationManager bgpConfigurationManager;
 251
 252     public BgpConfigurationManager(final DataBroker dataBroker,
 253             final EntityOwnershipService entityOwnershipService,
 254             final FibDSWriter fibDSWriter,
 255             final BundleContext bundleContext)
 256             throws InterruptedException, ExecutionException, TimeoutException {
 257         BgpConfigurationManager.dataBroker = dataBroker;
 258         BgpConfigurationManager.fibDSWriter = fibDSWriter;
 259         this.bundleContext = bundleContext;
 260         String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
 261         cHostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 262         cPortStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 263         LOG.info("UpdateServer at localhost:" + updatePort + " ConfigServer at "
 264                 + cHostStartup + ":" + cPortStartup);
 265         VtyshCli.setHostAddr(cHostStartup);
 266         ClearBgpCli.setHostAddr(cHostStartup);
 267         setEntityOwnershipService(entityOwnershipService);
 268         bgpRouter = BgpRouter.getInstance();
 269         odlThriftIp = readThriftIpForCommunication(SDNC_BGP_MIP);
 270         bgpThriftIp = readThriftIpForCommunication(BGP_SDNC_MIP);
 271         registerCallbacks();
 272
 273         LOG.info("BGP Configuration manager initialized");
 274         initer.countDown();
 275
 276         bgpConfigurationManager = this;
 277         BgpUtil.batchSize = BgpUtil.BATCH_SIZE;
 278         if (Integer.getInteger("batch.size") != null) {
 279             BgpUtil.batchSize = Integer.getInteger("batch.size");
 280         }
 281         BgpUtil.batchInterval = BgpUtil.PERIODICITY;
 282         if (Integer.getInteger("batch.wait.time") != null) {
 283             BgpUtil.batchInterval = Integer.getInteger("batch.wait.time");
 284         }
 285         BgpUtil.registerWithBatchManager(
 286                 new DefaultBatchHandler(dataBroker, LogicalDatastoreType.CONFIGURATION, BgpUtil.batchSize,
 287                         BgpUtil.batchInterval));
 288
 289         GlobalEventExecutor.INSTANCE.execute(() -> {
 290             final WaitingServiceTracker<IBgpManager> tracker = WaitingServiceTracker.create(
 291                     IBgpManager.class, bundleContext);
 292             bgpManager = tracker.waitForService(WaitingServiceTracker.FIVE_MINUTES);
 293             updateServer = new BgpThriftService(Integer.parseInt(updatePort), bgpManager, fibDSWriter);
 294             updateServer.start();
 295             LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 296         });
 297     }
 298
 299     private Object createListener(Class<?> cls) {
 300         Constructor<?> ctor;
 301         Object obj = null;
 302
 303         try {
 304             ctor = cls.getConstructor(BgpConfigurationManager.class);
 305             obj = ctor.newInstance(this);
 306         } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException
 307                 e) {
 308             LOG.error("Failed to create listener object", e);
 309         }
 310         return obj;
 311     }
 312
 313     private void registerCallbacks() {
 314         String emsg = "Failed to register listener";
 315         registrations = new ListenerRegistration[REACTORS.length];
 316         InstanceIdentifier<?> iid = InstanceIdentifier.create(Bgp.class);
 317         for (Class reactor : REACTORS) {
 318             Object obj = createListener(reactor);
 319             AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 320             dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 321         }
 322     }
 323
 324     public void close() {
 325         if (updateServer != null) {
 326             updateServer.stop();
 327         }
 328         LOG.info("{} close", getClass().getSimpleName());
 329     }
 330
 331     private boolean configExists() throws ReadFailedException {
 332         InstanceIdentifier.InstanceIdentifierBuilder<Bgp> iib =
 333                 InstanceIdentifier.builder(Bgp.class);
 334         InstanceIdentifier<Bgp> iid = iib.build();
 335         return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 336                 iid).isPresent();
 337     }
 338
 339     private String getProperty(String var, String def) {
 340         String property = bundleContext.getProperty(var);
 341         return (property == null ? def : property);
 342     }
 343
 344     boolean ignoreClusterDcnEventForFollower() {
 345         return !EntityOwnerUtils.amIEntityOwner(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME);
 346     }
 347
 348     public Bgp get() {
 349         config = getConfig();
 350         return config;
 351     }
 352
 353     public void setEntityOwnershipService(final EntityOwnershipService entityOwnershipService) {
 354         try {
 355             EntityOwnerUtils.registerEntityCandidateForOwnerShip(entityOwnershipService,
 356                 BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME, ownershipChange -> {
 357                     LOG.trace("entity owner change event fired");
 358                     if (ownershipChange.hasOwner() && ownershipChange.isOwner()) {
 359                         LOG.trace("This PL is the Owner");
 360                         activateMIP();
 361                         bgpRestarted();
 362                     } else {
 363                         LOG.info("Not owner: hasOwner: {}, isOwner: {}", ownershipChange.hasOwner(),
 364                                 ownershipChange.isOwner());
 365                     }
 366                 });
 367         } catch (CandidateAlreadyRegisteredException e) {
 368             LOG.error("failed to register bgp entity", e);
 369         }
 370     }
 371
 372     private static final String ADD_WARN =
 373             "Config store updated; undo with Delete if needed.";
 374     private static final String DEL_WARN =
 375             "Config store updated; undo with Add if needed.";
 376     private static final String UPD_WARN =
 377             "Update operation not supported; Config store updated;"
 378                     + " restore with another Update if needed.";
 379
 380     public class ConfigServerReactor
 381             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 382             implements AutoCloseable, ClusteredDataTreeChangeListener<ConfigServer> {
 383         private static final String YANG_OBJ = "config-server ";
 384
 385         public ConfigServerReactor() {
 386             super(ConfigServer.class, ConfigServerReactor.class);
 387         }
 388
 389         @Override
 390         protected synchronized void add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 391             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 392             if (ignoreClusterDcnEventForFollower()) {
 393                 return;
 394             }
 395
 396             try {
 397                 initer.await();
 398             } catch (InterruptedException e) {
 399                 // Ignored
 400             }
 401             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 402             synchronized (BgpConfigurationManager.this) {
 403                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 404                         val.getPort().intValue());
 405                 if (!res) {
 406                     LOG.error(YANG_OBJ + "Add failed; " + ADD_WARN);
 407                 }
 408             }
 409         }
 410
 411         @Override
 412         protected ConfigServerReactor getDataTreeChangeListener() {
 413             return ConfigServerReactor.this;
 414         }
 415
 416         @Override
 417         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 418             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 419         }
 420
 421         @Override
 422         protected synchronized void remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 423             LOG.trace("received bgp disconnect");
 424             if (ignoreClusterDcnEventForFollower()) {
 425                 return;
 426             }
 427             synchronized (BgpConfigurationManager.this) {
 428                 bgpRouter.disconnect();
 429             }
 430         }
 431
 432         @Override
 433         protected void update(InstanceIdentifier<ConfigServer> iid,
 434                 ConfigServer oldval, ConfigServer newval) {
 435             LOG.trace("received bgp Connection update");
 436             if (ignoreClusterDcnEventForFollower()) {
 437                 return;
 438             }
 439             LOG.error(YANG_OBJ + UPD_WARN);
 440         }
 441     }
 442
 443     private BgpRouter getClient(String yangObj) {
 444         if (bgpRouter == null || !bgpRouter.isBgpConnected()) {
 445             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 446             return null;
 447         }
 448         return bgpRouter;
 449     }
 450
 451     public class AsIdReactor
 452             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 453             implements AutoCloseable, ClusteredDataTreeChangeListener<AsId> {
 454
 455         private static final String YANG_OBJ = "as-id ";
 456
 457         public AsIdReactor() {
 458             super(AsId.class, AsIdReactor.class);
 459         }
 460
 461         @Override
 462         protected synchronized void add(InstanceIdentifier<AsId> iid, AsId val) {
 463             LOG.error("received bgp add asid {}", val);
 464             if (ignoreClusterDcnEventForFollower()) {
 465                 return;
 466             }
 467             LOG.debug("received add router config asNum {}", val.getLocalAs());
 468             synchronized (BgpConfigurationManager.this) {
 469                 IpAddress routerId = val.getRouterId();
 470                 BgpRouter br = getClient(YANG_OBJ);
 471                 long asNum = val.getLocalAs();
 472                 if (br == null) {
 473                     LOG.error("{} Unable to process add for routerId {} asNum {}; {}", YANG_OBJ, routerId, asNum,
 474                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 475                     return;
 476                 }
 477                 Boolean afb = val.isAnnounceFbit();
 478                 String rid = (routerId == null) ? "" : new String(routerId.getValue());
 479                 int stalepathTime = (int) getStalePathtime(RESTART_DEFAULT_GR, val);
 480                 boolean announceFbit = true;
 481                 try {
 482                     br.startBgp(asNum, rid, stalepathTime, announceFbit);
 483                     if (getBgpCounters() == null) {
 484                         startBgpCountersTask();
 485                     }
 486                     if (getBgpAlarms() == null) {
 487                         startBgpAlarmsTask();
 488                     }
 489                 } catch (BgpRouterException bre) {
 490                     if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
 491                         LOG.error(YANG_OBJ + "Add requested when BGP is already active");
 492                     } else {
 493                         LOG.error(YANG_OBJ + "Add received exception: \""
 494                                 + bre + "\"; " + ADD_WARN);
 495                     }
 496                 } catch (TException e) {
 497                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 498                 }
 499             }
 500         }
 501
 502         @Override
 503         protected AsIdReactor getDataTreeChangeListener() {
 504             return AsIdReactor.this;
 505         }
 506
 507         @Override
 508         protected InstanceIdentifier<AsId> getWildCardPath() {
 509             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 510         }
 511
 512         @Override
 513         protected synchronized void remove(InstanceIdentifier<AsId> iid, AsId val) {
 514             LOG.error("received delete router config asNum {}", val.getLocalAs());
 515             if (ignoreClusterDcnEventForFollower()) {
 516                 return;
 517             }
 518             synchronized (BgpConfigurationManager.this) {
 519                 long asNum = val.getLocalAs();
 520                 BgpRouter br = getClient(YANG_OBJ);
 521                 if (br == null) {
 522                     LOG.error("{} Unable to process remove for asNum {}; {}", YANG_OBJ, asNum,
 523                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 524                     return;
 525                 }
 526                 try {
 527                     br.stopBgp(asNum);
 528                 } catch (TException | BgpRouterException e) {
 529                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 530                 }
 531                 if (getBgpCounters() != null) {
 532                     stopBgpCountersTask();
 533                 }
 534                 if (getBgpAlarms() != null) {
 535                     stopBgpAlarmsTask();
 536                 }
 537             }
 538         }
 539
 540         @Override
 541         protected void update(InstanceIdentifier<AsId> iid,
 542                 AsId oldval, AsId newval) {
 543             if (ignoreClusterDcnEventForFollower()) {
 544                 return;
 545             }
 546             LOG.error(YANG_OBJ + UPD_WARN);
 547         }
 548     }
 549
 550     public class GracefulRestartReactor
 551             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 552             implements AutoCloseable, ClusteredDataTreeChangeListener<GracefulRestart> {
 553
 554         private static final String YANG_OBJ = "graceful-restart ";
 555
 556         public GracefulRestartReactor() {
 557             super(GracefulRestart.class, GracefulRestartReactor.class);
 558         }
 559
 560         @Override
 561         protected synchronized void add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 562             if (ignoreClusterDcnEventForFollower()) {
 563                 return;
 564             }
 565             synchronized (BgpConfigurationManager.this) {
 566                 int stalePathTime = val.getStalepathTime().intValue();
 567                 BgpRouter br = getClient(YANG_OBJ);
 568                 if (br == null) {
 569                     LOG.error("{} Unable to add stale-path time {}; {}", YANG_OBJ, stalePathTime,
 570                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 571                     return;
 572                 }
 573                 try {
 574                     br.addGracefulRestart(stalePathTime);
 575                 } catch (TException | BgpRouterException e) {
 576                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 577                 }
 578             }
 579         }
 580
 581         @Override
 582         protected GracefulRestartReactor getDataTreeChangeListener() {
 583             return GracefulRestartReactor.this;
 584         }
 585
 586         @Override
 587         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 588             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 589         }
 590
 591         @Override
 592         protected synchronized void remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 593             if (ignoreClusterDcnEventForFollower()) {
 594                 return;
 595             }
 596             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 597             synchronized (BgpConfigurationManager.this) {
 598                 BgpRouter br = getClient(YANG_OBJ);
 599                 if (br == null) {
 600                     LOG.error("{} Unable to delete stale-path time; {}", YANG_OBJ,
 601                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 602                     return;
 603                 }
 604                 try {
 605                     br.delGracefulRestart();
 606                 } catch (TException | BgpRouterException e) {
 607                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 608                 }
 609             }
 610         }
 611
 612         @Override
 613         protected void update(InstanceIdentifier<GracefulRestart> iid,
 614                 GracefulRestart oldval, GracefulRestart newval) {
 615             if (ignoreClusterDcnEventForFollower()) {
 616                 return;
 617             }
 618             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 619             synchronized (BgpConfigurationManager.this) {
 620                 int stalePathTime = newval.getStalepathTime().intValue();
 621                 BgpRouter br = getClient(YANG_OBJ);
 622                 if (br == null) {
 623                     LOG.error("{} Unable to update stale-path time to {}; {}", YANG_OBJ, stalePathTime,
 624                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 625                     return;
 626                 }
 627                 try {
 628                     br.addGracefulRestart(stalePathTime);
 629                 } catch (TException | BgpRouterException e) {
 630                     LOG.error("{} update received exception; {}", YANG_OBJ, ADD_WARN, e);
 631                 }
 632             }
 633         }
 634     }
 635
 636     public class LoggingReactor
 637             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 638             implements AutoCloseable, ClusteredDataTreeChangeListener<Logging> {
 639
 640         private static final String YANG_OBJ = "logging ";
 641
 642         public LoggingReactor() {
 643             super(Logging.class, LoggingReactor.class);
 644         }
 645
 646         @Override
 647         protected synchronized void add(InstanceIdentifier<Logging> iid, Logging val) {
 648             if (ignoreClusterDcnEventForFollower()) {
 649                 return;
 650             }
 651             synchronized (BgpConfigurationManager.this) {
 652                 BgpRouter br = getClient(YANG_OBJ);
 653                 if (br == null) {
 654                     LOG.error("{} Unable to add logging for qbgp; {}", YANG_OBJ,
 655                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 656                     return;
 657                 }
 658                 try {
 659                     br.setLogging(val.getFile(), val.getLevel());
 660                 } catch (TException | BgpRouterException e) {
 661                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 662                 }
 663             }
 664         }
 665
 666         @Override
 667         protected LoggingReactor getDataTreeChangeListener() {
 668             return LoggingReactor.this;
 669         }
 670
 671         @Override
 672         protected InstanceIdentifier<Logging> getWildCardPath() {
 673             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 674         }
 675
 676         @Override
 677         protected synchronized void remove(InstanceIdentifier<Logging> iid, Logging val) {
 678             if (ignoreClusterDcnEventForFollower()) {
 679                 return;
 680             }
 681             LOG.debug("received remove Logging config val {}", val.getLevel());
 682             synchronized (BgpConfigurationManager.this) {
 683                 BgpRouter br = getClient(YANG_OBJ);
 684                 if (br == null) {
 685                     LOG.error("{} Unable to remove logging for qbgp; {}", YANG_OBJ,
 686                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 687                     return;
 688                 }
 689                 try {
 690                     br.setLogging(DEF_LOGFILE, DEF_LOGLEVEL);
 691                 } catch (TException | BgpRouterException e) {
 692                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 693                 }
 694             }
 695         }
 696
 697         @Override
 698         protected void update(InstanceIdentifier<Logging> iid,
 699                 Logging oldval, Logging newval) {
 700             if (ignoreClusterDcnEventForFollower()) {
 701                 return;
 702             }
 703             synchronized (BgpConfigurationManager.this) {
 704                 BgpRouter br = getClient(YANG_OBJ);
 705                 if (br == null) {
 706                     LOG.error("{} Unable to update logging for qbgp; {}", YANG_OBJ,
 707                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 708                     return;
 709                 }
 710                 try {
 711                     br.setLogging(newval.getFile(), newval.getLevel());
 712                 } catch (TException | BgpRouterException e) {
 713                     LOG.error("{} newval received exception; {}", YANG_OBJ, ADD_WARN, e);
 714                 }
 715             }
 716         }
 717     }
 718
 719     public class NeighborsReactor
 720             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 721             implements AutoCloseable, ClusteredDataTreeChangeListener<Neighbors> {
 722
 723         private static final String YANG_OBJ = "neighbors ";
 724
 725         public NeighborsReactor() {
 726             super(Neighbors.class, NeighborsReactor.class);
 727         }
 728
 729         @Override
 730         protected synchronized void add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 731             if (ignoreClusterDcnEventForFollower()) {
 732                 return;
 733             }
 734             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 735             synchronized (BgpConfigurationManager.this) {
 736                 String peerIp = val.getAddress().getValue();
 737                 long as = val.getRemoteAs();
 738                 BgpRouter br = getClient(YANG_OBJ);
 739                 if (br == null) {
 740                     LOG.error("{} Unable to process add for peer {} as {}; {}", YANG_OBJ, peerIp, as,
 741                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 742                     return;
 743                 }
 744                 try {
 745                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 746                     br.addNeighbor(peerIp, as);
 747
 748                 } catch (TException | BgpRouterException e) {
 749                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 750                 }
 751             }
 752         }
 753
 754         @Override
 755         protected NeighborsReactor getDataTreeChangeListener() {
 756             return NeighborsReactor.this;
 757         }
 758
 759         @Override
 760         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 761             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class);
 762         }
 763
 764         @Override
 765         protected synchronized void remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 766             if (ignoreClusterDcnEventForFollower()) {
 767                 return;
 768             }
 769             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 770             synchronized (BgpConfigurationManager.this) {
 771                 String peerIp = val.getAddress().getValue();
 772                 BgpRouter br = getClient(YANG_OBJ);
 773                 if (br == null) {
 774                     LOG.error("{} Unable to process remove for peer {}; {}", YANG_OBJ, peerIp,
 775                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 776                     return;
 777                 }
 778                 try {
 779                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 780                     br.delNeighbor(peerIp);
 781                 } catch (TException | BgpRouterException e) {
 782                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 783                 }
 784             }
 785         }
 786
 787         @Override
 788         protected void update(InstanceIdentifier<Neighbors> iid,
 789                 Neighbors oldval, Neighbors newval) {
 790             if (ignoreClusterDcnEventForFollower()) {
 791                 return;
 792             }
 793             //purposefully nothing to do.
 794         }
 795     }
 796
 797     public class EbgpMultihopReactor
 798             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 799             implements AutoCloseable, ClusteredDataTreeChangeListener<EbgpMultihop> {
 800
 801         private static final String YANG_OBJ = "ebgp-multihop ";
 802
 803         public EbgpMultihopReactor() {
 804             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 805         }
 806
 807         @Override
 808         protected synchronized void add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 809             if (ignoreClusterDcnEventForFollower()) {
 810                 return;
 811             }
 812             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 813             synchronized (BgpConfigurationManager.this) {
 814                 String peerIp = val.getPeerIp().getValue();
 815                 BgpRouter br = getClient(YANG_OBJ);
 816                 if (br == null) {
 817                     LOG.error("{} Unable to process add for peer {}; {}", YANG_OBJ, peerIp,
 818                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 819                     return;
 820                 }
 821                 try {
 822                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 823                 } catch (TException | BgpRouterException e) {
 824                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 825                 }
 826             }
 827         }
 828
 829         @Override
 830         protected EbgpMultihopReactor getDataTreeChangeListener() {
 831             return EbgpMultihopReactor.this;
 832         }
 833
 834         @Override
 835         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
 836             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(EbgpMultihop.class);
 837         }
 838
 839         @Override
 840         protected synchronized void remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 841             if (ignoreClusterDcnEventForFollower()) {
 842                 return;
 843             }
 844             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 845             synchronized (BgpConfigurationManager.this) {
 846                 String peerIp = val.getPeerIp().getValue();
 847                 BgpRouter br = getClient(YANG_OBJ);
 848                 if (br == null) {
 849                     LOG.error("{} Unable to process remove for peer {}; {}", YANG_OBJ, peerIp,
 850                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 851                     return;
 852                 }
 853                 try {
 854                     br.delEbgpMultihop(peerIp);
 855                 } catch (TException | BgpRouterException e) {
 856                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 857                 }
 858             }
 859         }
 860
 861         @Override
 862         protected void update(InstanceIdentifier<EbgpMultihop> iid,
 863                 EbgpMultihop oldval, EbgpMultihop newval) {
 864             if (ignoreClusterDcnEventForFollower()) {
 865                 return;
 866             }
 867             LOG.error(YANG_OBJ + UPD_WARN);
 868         }
 869     }
 870
 871     public class UpdateSourceReactor
 872             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
 873             implements AutoCloseable, ClusteredDataTreeChangeListener<UpdateSource> {
 874
 875         private static final String YANG_OBJ = "update-source ";
 876
 877         public UpdateSourceReactor() {
 878             super(UpdateSource.class, UpdateSourceReactor.class);
 879         }
 880
 881         @Override
 882         protected synchronized void add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 883             if (ignoreClusterDcnEventForFollower()) {
 884                 return;
 885             }
 886             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
 887             synchronized (BgpConfigurationManager.this) {
 888                 String peerIp = val.getPeerIp().getValue();
 889                 BgpRouter br = getClient(YANG_OBJ);
 890                 if (br == null) {
 891                     LOG.error("{} Unable to process add for peer {}; {}", YANG_OBJ, peerIp,
 892                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 893                     return;
 894                 }
 895                 try {
 896                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
 897                 } catch (TException | BgpRouterException e) {
 898                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 899                 }
 900             }
 901         }
 902
 903         @Override
 904         protected UpdateSourceReactor getDataTreeChangeListener() {
 905             return UpdateSourceReactor.this;
 906         }
 907
 908         @Override
 909         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
 910             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(UpdateSource.class);
 911         }
 912
 913         @Override
 914         protected synchronized void remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 915             if (ignoreClusterDcnEventForFollower()) {
 916                 return;
 917             }
 918             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
 919             synchronized (BgpConfigurationManager.this) {
 920                 String peerIp = val.getPeerIp().getValue();
 921                 BgpRouter br = getClient(YANG_OBJ);
 922                 if (br == null) {
 923                     LOG.error("{} Unable to process remove for peer {}; {}", YANG_OBJ, peerIp,
 924                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 925                     return;
 926                 }
 927                 try {
 928                     br.delUpdateSource(peerIp);
 929                 } catch (TException | BgpRouterException e) {
 930                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 931                 }
 932             }
 933         }
 934
 935         @Override
 936         protected void update(InstanceIdentifier<UpdateSource> iid,
 937                 UpdateSource oldval, UpdateSource newval) {
 938             if (ignoreClusterDcnEventForFollower()) {
 939                 return;
 940             }
 941             LOG.error(YANG_OBJ + UPD_WARN);
 942         }
 943     }
 944
 945     public class AddressFamiliesReactor
 946             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
 947             implements AutoCloseable, ClusteredDataTreeChangeListener<AddressFamilies> {
 948
 949         private static final String YANG_OBJ = "address-families ";
 950
 951         public AddressFamiliesReactor() {
 952             super(AddressFamilies.class, AddressFamiliesReactor.class);
 953         }
 954
 955         @Override
 956         protected synchronized void add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
 957             if (ignoreClusterDcnEventForFollower()) {
 958                 return;
 959             }
 960             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
 961             synchronized (BgpConfigurationManager.this) {
 962                 String peerIp = val.getPeerIp().getValue();
 963                 BgpRouter br = getClient(YANG_OBJ);
 964                 if (br == null) {
 965                     LOG.error("{} Unable to process add for peer {}; {}", YANG_OBJ, peerIp,
 966                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 967                     return;
 968                 }
 969                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
 970                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
 971                 try {
 972                     br.addAddressFamily(peerIp, afi, safi);
 973                 } catch (TException | BgpRouterException e) {
 974                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 975                 }
 976             }
 977         }
 978
 979         @Override
 980         protected AddressFamiliesReactor getDataTreeChangeListener() {
 981             return AddressFamiliesReactor.this;
 982         }
 983
 984         @Override
 985         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
 986             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(AddressFamilies.class);
 987         }
 988
 989         @Override
 990         protected synchronized void remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
 991             if (ignoreClusterDcnEventForFollower()) {
 992                 return;
 993             }
 994             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
 995             synchronized (BgpConfigurationManager.this) {
 996                 String peerIp = val.getPeerIp().getValue();
 997                 BgpRouter br = getClient(YANG_OBJ);
 998                 if (br == null) {
 999                     LOG.error("{} Unable to process remove for peer {}; {}", YANG_OBJ, peerIp,
1000                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1001                     return;
1002                 }
1003                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1004                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1005                 try {
1006                     br.delAddressFamily(peerIp, afi, safi);
1007                 } catch (TException | BgpRouterException e) {
1008                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1009                 }
1010             }
1011         }
1012
1013         @Override
1014         protected void update(InstanceIdentifier<AddressFamilies> iid,
1015                 AddressFamilies oldval, AddressFamilies newval) {
1016             if (ignoreClusterDcnEventForFollower()) {
1017                 return;
1018             }
1019             LOG.error(YANG_OBJ + UPD_WARN);
1020         }
1021     }
1022
1023     public class NetworksReactor
1024             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
1025             implements AutoCloseable, ClusteredDataTreeChangeListener<Networks> {
1026
1027         private static final String YANG_OBJ = "networks ";
1028
1029         public NetworksReactor() {
1030             super(Networks.class, NetworksReactor.class);
1031         }
1032
1033         @Override
1034         public NetworksReactor getDataTreeChangeListener() {
1035             return NetworksReactor.this;
1036         }
1037
1038         @Override
1039         protected synchronized void add(InstanceIdentifier<Networks> iid, Networks val) {
1040             if (ignoreClusterDcnEventForFollower()) {
1041                 return;
1042             }
1043             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1044             synchronized (BgpConfigurationManager.this) {
1045                 String rd = val.getRd();
1046                 String pfxlen = val.getPrefixLen();
1047                 String nh = val.getNexthop().getValue();
1048                 BgpRouter br = getClient(YANG_OBJ);
1049                 if (br == null) {
1050                     LOG.error("{} Unable to process add for rd {} prefix {} nexthop {}; {}", YANG_OBJ, rd, pfxlen, nh,
1051                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1052                     return;
1053                 }
1054                 Long label = val.getLabel();
1055                 int lbl = (label == null) ? qbgpConstants.LBL_NO_LABEL
1056                         : label.intValue();
1057                 int l3vni = (val.getL3vni() == null) ? qbgpConstants.LBL_NO_LABEL
1058                         : val.getL3vni().intValue();
1059
1060                 BgpControlPlaneType protocolType = val.getBgpControlPlaneType();
1061                 int ethernetTag = val.getEthtag().intValue();
1062                 String esi = val.getEsi();
1063                 String macaddress = val.getMacaddress();
1064                 EncapType encapType = val.getEncapType();
1065                 String routerMac = val.getRoutermac();
1066
1067                 try {
1068                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, BgpUtil.convertToThriftProtocolType(protocolType),
1069                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1070                 } catch (TException | BgpRouterException e) {
1071                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1072                 }
1073             }
1074         }
1075
1076         @Override
1077         protected InstanceIdentifier<Networks> getWildCardPath() {
1078             return InstanceIdentifier.create(Bgp.class).child(Networks.class);
1079         }
1080
1081         @Override
1082         protected synchronized void remove(InstanceIdentifier<Networks> iid, Networks val) {
1083             if (ignoreClusterDcnEventForFollower()) {
1084                 return;
1085             }
1086             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1087             synchronized (BgpConfigurationManager.this) {
1088                 String rd = val.getRd();
1089                 String pfxlen = val.getPrefixLen();
1090                 BgpRouter br = getClient(YANG_OBJ);
1091                 if (br == null) {
1092                     LOG.error("{} Unable to process remove for rd {} prefix {}; {}", YANG_OBJ, rd, pfxlen,
1093                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1094                     return;
1095                 }
1096                 Long label = val.getLabel();
1097                 int lbl = (label == null) ? 0 : label.intValue();
1098                 if (rd == null && lbl > 0) {
1099                     //LU prefix is being deleted.
1100                     rd = Integer.toString(lbl);
1101                 }
1102                 try {
1103                     br.delPrefix(rd, pfxlen);
1104                 } catch (TException | BgpRouterException e) {
1105                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1106                 }
1107             }
1108         }
1109
1110         @Override
1111         protected void update(final InstanceIdentifier<Networks> iid,
1112                 final Networks oldval, final Networks newval) {
1113             if (ignoreClusterDcnEventForFollower()) {
1114                 return;
1115             }
1116             if (oldval.equals(newval)) {
1117                 //Update: OLD and New values are same, no need to trigger remove/add.
1118                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1119                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1120                 return;
1121             }
1122             LOG.debug("received update networks config val {}", newval.getPrefixLen());
1123             remove(iid, oldval);
1124             timer.schedule(new TimerTask() {
1125                 @Override
1126                 public void run() {
1127                     add(iid, newval);
1128                 }
1129             }, Integer.getInteger("bgp.nexthop.update.delay.in.secs", 5) * 1000);
1130         }
1131     }
1132
1133     static Timer timer = new Timer();
1134
1135     public class VrfsReactor
1136             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1137             implements AutoCloseable, ClusteredDataTreeChangeListener<Vrfs> {
1138
1139         private static final String YANG_OBJ = "vrfs ";
1140
1141         public VrfsReactor() {
1142             super(Vrfs.class, VrfsReactor.class);
1143         }
1144
1145         @Override
1146         protected synchronized void add(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1147             if (ignoreClusterDcnEventForFollower()) {
1148                 return;
1149             }
1150             LOG.debug("received add Vrfs config val {}", val.getRd());
1151             synchronized (BgpConfigurationManager.this) {
1152                 String rd = val.getRd();
1153                 BgpRouter br = getClient(YANG_OBJ);
1154                 if (br == null) {
1155                     LOG.error("{} Unable to process add for rd {}; {}", YANG_OBJ, rd,
1156                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1157                     return;
1158                 }
1159                 try {
1160                     br.addVrf(val.getLayerType(), rd, val.getImportRts(),
1161                             val.getExportRts());
1162                 } catch (TException | BgpRouterException e) {
1163                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1164                 }
1165             }
1166         }
1167
1168         @Override
1169         protected VrfsReactor getDataTreeChangeListener() {
1170             return VrfsReactor.this;
1171         }
1172
1173         @Override
1174         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1175             return InstanceIdentifier.create(Bgp.class).child(Vrfs.class);
1176         }
1177
1178         @Override
1179         protected synchronized void remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1180             if (ignoreClusterDcnEventForFollower()) {
1181                 return;
1182             }
1183             LOG.debug("received remove Vrfs config val {}", val.getRd());
1184             synchronized (BgpConfigurationManager.this) {
1185                 String rd = val.getRd();
1186                 BgpRouter br = getClient(YANG_OBJ);
1187                 if (br == null) {
1188                     LOG.error("{} Unable to process remove for rd {}; {}", YANG_OBJ, rd,
1189                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1190                     return;
1191                 }
1192                 try {
1193                     br.delVrf(rd);
1194                 } catch (TException | BgpRouterException e) {
1195                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1196                 }
1197             }
1198         }
1199
1200         @Override
1201         protected void update(InstanceIdentifier<Vrfs> iid,
1202                 Vrfs oldval, Vrfs newval) {
1203             if (ignoreClusterDcnEventForFollower()) {
1204                 return;
1205             }
1206             LOG.debug("VRFS: Update getting triggered for VRFS rd {}", oldval.getRd());
1207             LOG.error(YANG_OBJ + UPD_WARN);
1208         }
1209     }
1210
1211     Future lastCleanupJob;
1212     Future lastReplayJobFt = null;
1213
1214     protected void activateMIP() {
1215         try {
1216             LOG.trace("BgpReactor: Executing MIP Activate command");
1217             Process processBgp = Runtime.getRuntime().exec("cluster ip -a sdnc_bgp_mip");
1218             Process processOs = Runtime.getRuntime().exec("cluster ip -a sdnc_os_mip");
1219             LOG.trace("bgpMIP Activated");
1220
1221         } catch (IOException io) {
1222             LOG.error("IO Exception got while activating mip: {}", io.getMessage());
1223         }
1224     }
1225
1226     AtomicBoolean started = new AtomicBoolean(false);
1227
1228     public class BgpReactor
1229             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1230             implements AutoCloseable, ClusteredDataTreeChangeListener<Bgp> {
1231
1232         private static final String YANG_OBJ = "Bgp ";
1233
1234         public BgpReactor() {
1235             super(Bgp.class, BgpReactor.class);
1236         }
1237
1238
1239         @Override
1240         protected synchronized void add(InstanceIdentifier<Bgp> iid, Bgp val) {
1241             LOG.error("received add Bgp config replaying the config");
1242
1243             try {
1244                 initer.await();
1245             } catch (InterruptedException e) {
1246                 // Ignored
1247             }
1248             synchronized (BgpConfigurationManager.this) {
1249                 config = val;
1250                 if (ignoreClusterDcnEventForFollower()) {
1251                     return;
1252                 }
1253                 activateMIP();
1254                 if (isIpAvailable(odlThriftIp)) {
1255                     bgpRestarted();
1256                 } else {
1257                     IP_ACTIVATION_CHECK_TIMER.scheduleAtFixedRate(new TimerTask() {
1258                         @Override
1259                         public void run() {
1260                             if (isIpAvailable(odlThriftIp)) {
1261                                 bgpRestarted();
1262                                 IP_ACTIVATION_CHECK_TIMER.cancel();
1263                             } else {
1264                                 LOG.trace("waiting for odlThriftIP: {} to be present", odlThriftIp);
1265                             }
1266                         }
1267                     }, 10000L, 10000L);
1268                 }
1269             }
1270         }
1271
1272         @Override
1273         protected BgpReactor getDataTreeChangeListener() {
1274             return BgpReactor.this;
1275         }
1276
1277         @Override
1278         protected InstanceIdentifier<Bgp> getWildCardPath() {
1279             return InstanceIdentifier.create(Bgp.class);
1280         }
1281
1282         @Override
1283         protected synchronized void remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1284             if (ignoreClusterDcnEventForFollower()) {
1285                 return;
1286             }
1287             LOG.debug("received remove Bgp config");
1288             synchronized (BgpConfigurationManager.this) {
1289                 config = null;
1290             }
1291         }
1292
1293         @Override
1294         protected void update(InstanceIdentifier<Bgp> iid,
1295                 Bgp oldval, Bgp newval) {
1296             if (ignoreClusterDcnEventForFollower()) {
1297                 return;
1298             }
1299             synchronized (BgpConfigurationManager.this) {
1300                 config = newval;
1301             }
1302         }
1303     }
1304
1305     @SuppressWarnings("deprecation")
1306     public class MultipathReactor
1307             extends AsyncDataTreeChangeListenerBase<Multipath, MultipathReactor>
1308             implements AutoCloseable, ClusteredDataTreeChangeListener<Multipath> {
1309
1310         private static final String YANG_OBJ = "multipath ";
1311
1312         public MultipathReactor() {
1313             super(Multipath.class, MultipathReactor.class);
1314         }
1315
1316
1317         @Override
1318         protected MultipathReactor getDataTreeChangeListener() {
1319             return MultipathReactor.this;
1320         }
1321
1322         @Override
1323         protected InstanceIdentifier<Multipath> getWildCardPath() {
1324             return InstanceIdentifier.create(Bgp.class).child(Multipath.class);
1325         }
1326
1327         @Override
1328         protected synchronized void remove(InstanceIdentifier<Multipath> iid, Multipath val) {
1329             ExecutorService executor = Executors.newSingleThreadExecutor();
1330             executor.submit(new MultipathStatusChange(val));
1331             executor.shutdown();
1332         }
1333
1334         @Override
1335         protected void update(InstanceIdentifier<Multipath> iid, Multipath oldval, Multipath newval) {
1336             ExecutorService executor = Executors.newSingleThreadExecutor();
1337             executor.submit(new MultipathStatusChange(newval));
1338             executor.shutdown();
1339         }
1340
1341         @Override
1342         protected void add(InstanceIdentifier<Multipath> key, Multipath dataObjectModification) {
1343             ExecutorService executor = Executors.newSingleThreadExecutor();
1344             executor.submit(new MultipathStatusChange(dataObjectModification));
1345             executor.shutdown();
1346         }
1347
1348         class MultipathStatusChange implements Callable<Void> {
1349
1350             Multipath multipath;
1351
1352             MultipathStatusChange(Multipath multipath) {
1353                 this.multipath = multipath;
1354             }
1355
1356             @Override
1357             public Void call() throws Exception {
1358                 if (!ignoreClusterDcnEventForFollower()) {
1359                     synchronized (BgpConfigurationManager.this) {
1360
1361                         BgpRouter br = getClient(YANG_OBJ);
1362
1363                         if (br != null) {
1364                             af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1365                             af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1366
1367                             try {
1368                                 if (multipath.isMultipathEnabled()) {
1369                                     br.enableMultipath(afi, safi);
1370                                 } else {
1371                                     br.disableMultipath(afi, safi);
1372                                 }
1373                             } catch (TException | BgpRouterException e) {
1374                                 LOG.error(YANG_OBJ + " received exception: \"" + e + "\"");
1375                             }
1376                         }
1377                     }
1378                 }
1379                 return null;
1380             }
1381
1382         }
1383
1384         @Override
1385         public void close() {
1386             super.close();
1387         }
1388     }
1389
1390     @SuppressWarnings("deprecation")
1391     public class VrfMaxpathReactor
1392             extends AsyncDataTreeChangeListenerBase<VrfMaxpath, VrfMaxpathReactor>
1393             implements AutoCloseable, ClusteredDataTreeChangeListener<VrfMaxpath> {
1394
1395         private static final String YANG_OBJ = "vrfMaxpath ";
1396
1397         public VrfMaxpathReactor() {
1398             super(VrfMaxpath.class, VrfMaxpathReactor.class);
1399         }
1400
1401
1402         @Override
1403         protected VrfMaxpathReactor getDataTreeChangeListener() {
1404             return VrfMaxpathReactor.this;
1405         }
1406
1407         @Override
1408         protected InstanceIdentifier<VrfMaxpath> getWildCardPath() {
1409             return InstanceIdentifier.create(Bgp.class).child(VrfMaxpath.class);
1410         }
1411
1412         class VrfMaxPathConfigurator implements Callable<Void> {
1413
1414             VrfMaxpath vrfMaxpathVal;
1415
1416             VrfMaxPathConfigurator(VrfMaxpath vrfMaxPathVal) {
1417                 this.vrfMaxpathVal = vrfMaxPathVal;
1418             }
1419
1420             @Override
1421             public Void call() throws Exception {
1422                 if (!ignoreClusterDcnEventForFollower()) {
1423                     synchronized (BgpConfigurationManager.this) {
1424                         BgpRouter br = getClient(YANG_OBJ);
1425                         if (br != null) {
1426                             try {
1427                                 br.multipaths(vrfMaxpathVal.getRd(), vrfMaxpathVal.getMaxpaths());
1428                                 LOG.debug("Maxpath for vrf: " + vrfMaxpathVal.getRd() + " : is "
1429                                         + vrfMaxpathVal.getMaxpaths());
1430                             } catch (TException | BgpRouterException e) {
1431                                 LOG.error(YANG_OBJ
1432                                         + " received exception: \"" + e + "\"");
1433                             }
1434                         }
1435                     }
1436                 }
1437                 return null;
1438             }
1439         }
1440
1441         @Override
1442         protected synchronized void remove(InstanceIdentifier<VrfMaxpath> iid, VrfMaxpath vrfMaxPathVal) {
1443             ExecutorService executor = Executors.newSingleThreadExecutor();
1444             executor.submit(new VrfMaxPathConfigurator(vrfMaxPathVal));
1445             executor.shutdown();
1446         }
1447
1448         @Override
1449         protected void update(InstanceIdentifier<VrfMaxpath> iid,
1450                               VrfMaxpath oldval, VrfMaxpath newval) {
1451             if (oldval.getMaxpaths() != newval.getMaxpaths()) {
1452                 ExecutorService executor = Executors.newSingleThreadExecutor();
1453                 executor.submit(new VrfMaxPathConfigurator(newval));
1454                 executor.shutdown();
1455             }
1456         }
1457
1458         @Override
1459         protected void add(InstanceIdentifier<VrfMaxpath> instanceIdentifier, VrfMaxpath vrfMaxpathVal) {
1460             ExecutorService executor = Executors.newSingleThreadExecutor();
1461             executor.submit(new VrfMaxPathConfigurator(vrfMaxpathVal));
1462             executor.shutdown();
1463         }
1464
1465         @Override
1466         public void close() {
1467             super.close();
1468         }
1469     }
1470
1471     public String readThriftIpForCommunication(String mipAddr) {
1472         File file = new File(CLUSTER_CONF_FILE);
1473         if (!file.exists()) {
1474             return DEF_CHOST;
1475         }
1476
1477         try (BufferedReader br = new BufferedReader(new FileReader(file))) {
1478             String line;
1479             while ((line = br.readLine()) != null) {
1480                 if (line.contains(mipAddr)) {
1481                     line = line.trim();
1482                     return line.substring(line.lastIndexOf(" ") + 1);
1483                 }
1484             }
1485         } catch (FileNotFoundException e) {
1486             return DEF_CHOST;
1487         } catch (IOException e) {
1488             LOG.error("Error reading {}", CLUSTER_CONF_FILE, e);
1489         }
1490         return DEF_CHOST;
1491     }
1492
1493     public boolean isIpAvailable(String odlip) {
1494
1495         try {
1496             if (odlip != null) {
1497                 if ("127.0.0.1".equals(odlip)) {
1498                     return true;
1499                 }
1500                 Enumeration networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces();
1501                 while (networkInterfaceEnumeration.hasMoreElements()) {
1502                     NetworkInterface networkInterface = (NetworkInterface) networkInterfaceEnumeration.nextElement();
1503                     Enumeration inetAddressEnumeration = networkInterface.getInetAddresses();
1504                     while (inetAddressEnumeration.hasMoreElements()) {
1505                         InetAddress inetAddress = (InetAddress) inetAddressEnumeration.nextElement();
1506                         if (odlip.equals(inetAddress.getHostAddress())) {
1507                             return true;
1508                         }
1509                     }
1510                 }
1511             }
1512         } catch (SocketException e) {
1513             // Ignored?
1514         }
1515         return false;
1516     }
1517
1518     public static long getStalePathtime(int defValue, AsId asId) {
1519         long spt = 0;
1520         try {
1521             spt = getConfig().getGracefulRestart().getStalepathTime();
1522         } catch (NullPointerException e) {
1523             try {
1524                 spt = asId.getStalepathTime();
1525                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1526             } catch (NullPointerException ignore) {
1527                 LOG.trace("BGP AS id is not set using graceful");
1528                 spt = defValue;
1529             }
1530         }
1531         if (spt == 0) {
1532             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1533             spt = defValue;
1534         }
1535         return spt;
1536     }
1537
1538     public synchronized void bgpRestarted() {
1539         /*
1540          * If there a thread which in the process of stale cleanup, cancel it
1541          * and start a new thread (to avoid processing same again).
1542          */
1543         if (previousReplayJobInProgress()) {
1544             cancelPreviousReplayJob();
1545         }
1546         Runnable task = () -> {
1547             try {
1548                 LOG.info("running bgp replay task ");
1549                 if (get() == null) {
1550                     String host = getConfigHost();
1551                     int port = getConfigPort();
1552                     LOG.info("connecting  to bgp host {} ", host);
1553
1554                     boolean res = bgpRouter.connect(host, port);
1555                     LOG.info("no config to push in bgp replay task ");
1556                     return;
1557                 }
1558                 setStaleStartTime(System.currentTimeMillis());
1559                 LOG.info("started creating stale fibDSWriter  map ");
1560                 createStaleFibMap();
1561                 setStaleEndTime(System.currentTimeMillis());
1562                 LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
1563                 LOG.info("started bgp config replay ");
1564                 setCfgReplayStartTime(System.currentTimeMillis());
1565                 try {
1566                     replay();
1567                 } catch (TimeoutException | ExecutionException e) {
1568                     LOG.error("Error while replaying routes. {}", e);
1569                 }
1570                 setCfgReplayEndTime(System.currentTimeMillis());
1571                 LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1572                 long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1573                 Thread.sleep(routeSyncTime * 1000L);
1574                 setStaleCleanupTime(routeSyncTime);
1575                 new RouteCleanup().call();
1576             } catch (InterruptedException eCancel) {
1577                 LOG.error("Stale Cleanup Task Cancelled", eCancel);
1578             }
1579         };
1580         lastReplayJobFt = executor.submit(task);
1581     }
1582
1583     private boolean previousReplayJobInProgress() {
1584         return lastReplayJobFt != null && !lastReplayJobFt.isDone();
1585     }
1586
1587     private void cancelPreviousReplayJob() {
1588         try {
1589             LOG.error("cancelling already running bgp replay task");
1590             lastReplayJobFt.cancel(true);
1591             lastReplayJobFt = null;
1592             Thread.sleep(2000);
1593         } catch (InterruptedException e) {
1594             LOG.error("Failed to cancel previous replay job ", e);
1595         }
1596     }
1597
1598     private static void doRouteSync() throws InterruptedException, TimeoutException, ExecutionException {
1599         BgpSyncHandle bsh = BgpSyncHandle.getInstance();
1600         LOG.error("Starting BGP route sync");
1601         try {
1602             bgpRouter.initRibSync(bsh);
1603         } catch (TException | BgpRouterException e) {
1604             LOG.error("Route sync aborted, exception when initializing", e);
1605             return;
1606         }
1607         while (bsh.getState() != bsh.DONE) {
1608             Routes routes = null;
1609             try {
1610                 routes = bgpRouter.doRibSync(bsh);
1611             } catch (TException | BgpRouterException e) {
1612                 LOG.error("Route sync aborted, exception when syncing", e);
1613                 return;
1614             }
1615             Iterator<Update> updates = routes.getUpdatesIterator();
1616             while (updates.hasNext()) {
1617                 Update update = updates.next();
1618                 Map<String, Map<String, Long>> staleFibRdMap = BgpConfigurationManager.getStaledFibEntriesMap();
1619                 String rd = update.getRd();
1620                 String nexthop = update.getNexthop();
1621
1622                 // TODO: decide correct label here
1623                 int label = update.getL3label();
1624
1625                 String prefix = update.getPrefix();
1626                 int plen = update.getPrefixlen();
1627
1628
1629                 // TODO: protocol type will not be available in "update"
1630                 // use "rd" to query vrf table and obtain the protocol_type. Currently using PROTOCOL_EVPN as default.
1631                 onUpdatePushRoute(
1632                         protocol_type.PROTOCOL_EVPN,
1633                         rd,
1634                         prefix,
1635                         plen,
1636                         nexthop,
1637                         update.getEthtag(),
1638                         update.getEsi(),
1639                         update.getMacaddress(),
1640                         label,
1641                         update.getRoutermac()
1642                 );
1643             }
1644         }
1645         try {
1646             LOG.error("Ending BGP route-sync");
1647             bgpRouter.endRibSync(bsh);
1648         } catch (TException | BgpRouterException e) {
1649             // Ignored?
1650         }
1651     }
1652
1653     /* onUpdatePushRoute
1654      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1655      *  - Entry compare shall include NextHop, Label.
1656      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1657      *  - If entry not found, add to FIB Config DS.
1658      *  - If entry found, but either Label/NextHop doesn't match.
1659      *      - Update FIB Config DS with modified values.
1660      *      - delete from Stale Map.
1661      */
1662
1663     public static void onUpdatePushRoute(protocol_type protocolType,
1664                                          String rd,
1665                                          String prefix,
1666                                          int plen,
1667                                          String nextHop,
1668                                          int ethtag,
1669                                          String esi,
1670                                          String macaddress,
1671                                          int label,
1672                                          String routermac)
1673             throws InterruptedException, ExecutionException, TimeoutException {
1674         boolean addroute = false;
1675         long l3vni = 0L;
1676         VrfEntry.EncapType encapType = VrfEntry.EncapType.Mplsgre;
1677         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1678             encapType = VrfEntry.EncapType.Vxlan;
1679             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = BgpUtil.getVpnInstanceOpData(dataBroker, rd);
1680             if (vpnInstanceOpDataEntry != null) {
1681                 l3vni = vpnInstanceOpDataEntry.getL3vni();
1682             } else {
1683                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1684                 return;
1685             }
1686         }
1687         if (!staledFibEntriesMap.isEmpty()) {
1688             // restart Scenario, as MAP is not empty.
1689             Map<String, Long> map = staledFibEntriesMap.get(rd);
1690             if (map != null) {
1691                 String prefixNextHop = appendNextHopToPrefix(prefix + "/" + plen, nextHop);
1692                 Long labelInStaleMap = map.get(prefixNextHop);
1693                 if (null == labelInStaleMap) {
1694                     // New Entry, which happened to be added during restart.
1695                     addroute = true;
1696                 } else {
1697                     map.remove(prefixNextHop);
1698                     if (isRouteModified(label, labelInStaleMap)) {
1699                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1700                         // Existing entry, where in Label got modified during restart
1701                         addroute = true;
1702                     }
1703                 }
1704             }
1705         } else {
1706             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1707             addroute = true;
1708         }
1709         if (addroute) {
1710             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1711             // TODO: modify addFibEntryToDS signature
1712             fibDSWriter.addFibEntryToDS(rd, macaddress, prefix + "/" + plen, Collections.singletonList(nextHop),
1713                     encapType, label, l3vni, routermac, RouteOrigin.BGP);
1714             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1715         }
1716     }
1717
1718     private static boolean isRouteModified(int label, Long labelInStaleMap) {
1719         return labelInStaleMap != null && !labelInStaleMap.equals(label);
1720     }
1721
1722     private static void replayNbrConfig(List<Neighbors> neighbors, BgpRouter br) {
1723         for (Neighbors nbr : neighbors) {
1724             try {
1725                 br.addNeighbor(nbr.getAddress().getValue(),
1726                         nbr.getRemoteAs());
1727                 //itmProvider.buildTunnelsToDCGW(new IpAddress(nbr.getAddress().getValue().toCharArray()));
1728             } catch (TException | BgpRouterException e) {
1729                 LOG.error("Replay:addNbr() received exception", e);
1730                 continue;
1731             }
1732             EbgpMultihop en = nbr.getEbgpMultihop();
1733             if (en != null) {
1734                 try {
1735                     br.addEbgpMultihop(en.getPeerIp().getValue(),
1736                             en.getNhops().intValue());
1737                 } catch (TException | BgpRouterException e) {
1738                     LOG.error("Replay:addEBgp() received exception", e);
1739                 }
1740             }
1741             UpdateSource us = nbr.getUpdateSource();
1742             if (us != null) {
1743                 try {
1744                     br.addUpdateSource(us.getPeerIp().getValue(),
1745                             us.getSourceIp().getValue());
1746                 } catch (TException | BgpRouterException e) {
1747                     LOG.error("Replay:addUS() received exception", e);
1748                 }
1749             }
1750             List<AddressFamilies> afs = nbr.getAddressFamilies();
1751             if (afs != null) {
1752                 for (AddressFamilies af : afs) {
1753                     af_afi afi = af_afi.findByValue(af.getAfi().intValue());
1754                     af_safi safi = af_safi.findByValue(af.getSafi().intValue());
1755                     try {
1756                         br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
1757                     } catch (TException | BgpRouterException e) {
1758                         LOG.error("Replay:addAf() received exception", e);
1759                     }
1760                 }
1761             }
1762         }
1763     }
1764
1765     public static String getConfigHost() {
1766         if (config == null) {
1767             return cHostStartup;
1768         }
1769         ConfigServer ts = config.getConfigServer();
1770         return (ts == null ? cHostStartup : ts.getHost().getValue());
1771     }
1772
1773     public static int getConfigPort() {
1774         if (config == null) {
1775             return Integer.parseInt(cPortStartup);
1776         }
1777         ConfigServer ts = config.getConfigServer();
1778         return (ts == null ? Integer.parseInt(cPortStartup) :
1779                 ts.getPort().intValue());
1780     }
1781
1782     public static Bgp getConfig() {
1783         AtomicInteger bgpDSretryCount = new AtomicInteger(DS_RETRY_COOUNT);
1784         while (0 != bgpDSretryCount.decrementAndGet()) {
1785             try {
1786                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
1787                         InstanceIdentifier.create(Bgp.class)).orNull();
1788             } catch (ReadFailedException e) {
1789                 //Config DS may not be up, so sleep for 1 second and retry
1790                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
1791                 try {
1792                     Thread.sleep(WAIT_TIME_BETWEEN_EACH_TRY_MILLIS);
1793                 } catch (InterruptedException timerEx) {
1794                     LOG.debug("WAIT_TIME_BETWEEN_EACH_TRY_MILLIS, Timer got interrupted while waiting for"
1795                             + "config DS availability", timerEx);
1796                 }
1797             }
1798         }
1799         LOG.error("failed to get bgp config");
1800         return null;
1801     }
1802
1803     @SuppressWarnings("checkstyle:IllegalCatch")
1804     public synchronized void replay() throws InterruptedException, TimeoutException, ExecutionException {
1805         synchronized (bgpConfigurationManager) {
1806             String host = getConfigHost();
1807             int port = getConfigPort();
1808             LOG.error("connecting  to bgp host {} ", host);
1809
1810             boolean res = bgpRouter.connect(host, port);
1811             if (!res) {
1812                 String msg = "Cannot connect to BGP config server at " + host + ":" + port;
1813                 if (config != null) {
1814                     msg += "; Configuration Replay aborted";
1815                 }
1816                 LOG.error(msg);
1817                 return;
1818             }
1819             config = getConfig();
1820             if (config == null) {
1821                 LOG.error("bgp config is empty nothing to push to bgp");
1822                 return;
1823             }
1824             BgpRouter br = bgpRouter;
1825             AsId asId = config.getAsId();
1826             if (asId == null) {
1827                 return;
1828             }
1829             long asNum = asId.getLocalAs();
1830             IpAddress routerId = asId.getRouterId();
1831             Long spt = asId.getStalepathTime();
1832             Boolean afb = asId.isAnnounceFbit();
1833             String rid = (routerId == null) ? "" : new String(routerId.getValue());
1834             int stalepathTime = (int) getStalePathtime(0, config.getAsId());
1835             boolean announceFbit = true;
1836             try {
1837                 br.startBgp(asNum, rid, stalepathTime, announceFbit);
1838             } catch (BgpRouterException bre) {
1839                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
1840                     doRouteSync();
1841                 } else {
1842                     LOG.error("Replay: startBgp() received exception: \""
1843                             + bre + "\"; " + ADD_WARN);
1844                 }
1845             } catch (TException e) {
1846                 //not unusual. We may have restarted & BGP is already on
1847                 LOG.error("Replay:startBgp() received exception: \"" + e + "\"");
1848             }
1849
1850             if (getBgpCounters() == null) {
1851                 startBgpCountersTask();
1852             }
1853
1854             if (getBgpAlarms() == null) {
1855                 startBgpAlarmsTask();
1856             }
1857
1858             Logging logging = config.getLogging();
1859             if (logging != null) {
1860                 try {
1861                     br.setLogging(logging.getFile(), logging.getLevel());
1862                 } catch (TException | BgpRouterException e) {
1863                     LOG.error("Replay:setLogging() received exception", e);
1864                 }
1865             }
1866
1867             GracefulRestart gracefulRestart = config.getGracefulRestart();
1868             if (gracefulRestart != null) {
1869                 try {
1870                     br.addGracefulRestart(gracefulRestart.getStalepathTime().intValue());
1871                 } catch (TException | BgpRouterException e) {
1872                     LOG.error("Replay:addGr() received exception", e);
1873                 }
1874             }
1875
1876             List<Vrfs> vrfs = config.getVrfs();
1877             if (vrfs != null) {
1878                 for (Vrfs vrf : vrfs) {
1879                     try {
1880                         br.addVrf(vrf.getLayerType(), vrf.getRd(), vrf.getImportRts(),
1881                                 vrf.getExportRts());
1882                     } catch (TException | BgpRouterException e) {
1883                         LOG.error("Replay:addVrf() received exception", e);
1884                     }
1885                 }
1886             }
1887
1888             List<Networks> ln = config.getNetworks();
1889             if (ln != null) {
1890                 for (Networks net : ln) {
1891                     String rd = net.getRd();
1892                     String pfxlen = net.getPrefixLen();
1893                     String nh = net.getNexthop().getValue();
1894                     Long label = net.getLabel();
1895                     int lbl = (label == null) ? 0 : label.intValue();
1896                     int l3vni = (net.getL3vni() == null) ? 0 : net.getL3vni().intValue();
1897                     if (rd == null && lbl > 0) {
1898                         //LU prefix is being deleted.
1899                         rd = Integer.toString(lbl);
1900                     }
1901
1902                     BgpControlPlaneType protocolType = net.getBgpControlPlaneType();
1903                     int ethernetTag = net.getEthtag().intValue();
1904                     String esi = net.getEsi();
1905                     String macaddress = net.getMacaddress();
1906                     EncapType encapType = net.getEncapType();
1907                     String routerMac = net.getRoutermac();
1908
1909                     try {
1910                         br.addPrefix(rd, pfxlen, nh, lbl, l3vni, BgpUtil.convertToThriftProtocolType(protocolType),
1911                                 ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1912                     } catch (Exception e) {
1913                         LOG.error("Replay:addPfx() received exception", e);
1914                     }
1915                 }
1916             }
1917             List<Neighbors> neighbors = config.getNeighbors();
1918             if (neighbors != null) {
1919                 LOG.error("configuring existing Neighbors present for replay total neighbors {}", neighbors.size());
1920                 replayNbrConfig(neighbors, br);
1921             } else {
1922                 LOG.error("no Neighbors present for replay config ");
1923             }
1924
1925             List<Multipath> multipaths = config.getMultipath();
1926
1927             if (multipaths != null) {
1928                 for (Multipath multipath: multipaths) {
1929                     if (multipath != null) {
1930                         af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1931                         af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1932
1933                         try {
1934                             if (multipath.isMultipathEnabled()) {
1935                                 br.enableMultipath(afi, safi);
1936                             } else {
1937                                 br.disableMultipath(afi, safi);
1938                             }
1939                         } catch (TException | BgpRouterException e) {
1940                             LOG.info("Replay:multipaths() received exception: \"" + e + "\"");
1941                         }
1942                     }
1943                 }
1944             }
1945             List<VrfMaxpath> vrfMaxpaths = config.getVrfMaxpath();
1946             if (vrfMaxpaths != null) {
1947                 for (VrfMaxpath vrfMaxpath: vrfMaxpaths) {
1948                     try {
1949                         br.multipaths(vrfMaxpath.getRd(), vrfMaxpath.getMaxpaths());
1950                     } catch (TException | BgpRouterException e) {
1951                         LOG.info("Replay:vrfMaxPath() received exception: \"" + e + "\"");
1952                     }
1953                 }
1954             }
1955         }
1956     }
1957
1958     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
1959         BgpUtil.update(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1960     }
1961
1962     private <T extends DataObject> void asyncWrite(InstanceIdentifier<T> iid, T dto) {
1963         BgpUtil.write(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1964     }
1965
1966     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
1967         BgpUtil.delete(dataBroker, LogicalDatastoreType.CONFIGURATION, iid);
1968     }
1969
1970     public synchronized void startConfig(String bgpHost, int thriftPort) {
1971         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
1972                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
1973         InstanceIdentifier<ConfigServer> iid = iib.build();
1974         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
1975         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
1976                 .setPort((long) thriftPort).build();
1977         update(iid, dto);
1978     }
1979
1980     public synchronized void startBgp(long as, String routerId, int spt, boolean fbit) {
1981         IpAddress rid = (routerId == null) ? null : new IpAddress(routerId.toCharArray());
1982         Long staleTime = (long) spt;
1983         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
1984                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
1985         InstanceIdentifier<AsId> iid = iib.build();
1986         AsId dto = new AsIdBuilder().setLocalAs(as)
1987                 .setRouterId(rid)
1988                 .setStalepathTime(staleTime)
1989                 .setAnnounceFbit(fbit).build();
1990         update(iid, dto);
1991     }
1992
1993     public synchronized void addLogging(String fileName, String logLevel) {
1994         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
1995                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
1996         InstanceIdentifier<Logging> iid = iib.build();
1997         Logging dto = new LoggingBuilder().setFile(fileName)
1998                 .setLevel(logLevel).build();
1999         update(iid, dto);
2000     }
2001
2002     public synchronized void addGracefulRestart(int staleTime) {
2003         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2004                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
2005         InstanceIdentifier<GracefulRestart> iid = iib.build();
2006         GracefulRestart dto = new GracefulRestartBuilder()
2007                 .setStalepathTime((long) staleTime).build();
2008         update(iid, dto);
2009     }
2010
2011     public synchronized void addNeighbor(
2012             String nbrIp, long remoteAs, @Nullable final TcpMd5SignaturePasswordType md5Secret) {
2013         addNeighborAux(nbrIp, remoteAs, md5Secret);
2014     } // public addNeighbor(nbrIp, remoteAs, md5Secret)
2015
2016     public synchronized void addNeighbor(String nbrIp, long remoteAs) {
2017         addNeighborAux(nbrIp, remoteAs, null);
2018     } // public addNeighbor(nbrIp, remoteAs)
2019
2020     private void addNeighborAux(String nbrIp, long remoteAs, @Nullable final TcpMd5SignaturePasswordType md5Secret) {
2021         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2022         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2023                 InstanceIdentifier.builder(Bgp.class)
2024                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2025         InstanceIdentifier<Neighbors> iid = iib.build();
2026         TcpSecurityOption tcpSecOption = null;
2027         if (md5Secret != null) {
2028             tcpSecOption = new TcpMd5SignatureOptionBuilder().setTcpMd5SignaturePassword(md5Secret).build();
2029         } // else let tcpSecOption be null
2030         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
2031                 .setRemoteAs(remoteAs).setTcpSecurityOption(tcpSecOption).build();
2032         update(iid, dto);
2033     } // private addNeighborAux(nbrIp, remoteAs, md5Secret)
2034
2035     public synchronized void addUpdateSource(String nbrIp, String srcIp) {
2036         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2037         Ipv4Address srcAddr = new Ipv4Address(srcIp);
2038         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2039                 InstanceIdentifier.builder(Bgp.class)
2040                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2041                         .child(UpdateSource.class);
2042         InstanceIdentifier<UpdateSource> iid = iib.build();
2043         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
2044                 .setSourceIp(srcAddr).build();
2045         update(iid, dto);
2046     }
2047
2048     public synchronized void addEbgpMultihop(String nbrIp, int hops) {
2049         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2050         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2051                 InstanceIdentifier.builder(Bgp.class)
2052                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2053                         .child(EbgpMultihop.class);
2054         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2055         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
2056                 .setNhops((long) hops).build();
2057         update(iid, dto);
2058     }
2059
2060     public synchronized void addAddressFamily(String nbrIp, int afi, int safi) {
2061         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2062         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2063                 InstanceIdentifier.builder(Bgp.class)
2064                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2065                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2066         InstanceIdentifier<AddressFamilies> iid = iib.build();
2067         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
2068                 .setAfi((long) afi).setSafi((long) safi).build();
2069         update(iid, dto);
2070     }
2071
2072     public synchronized void addPrefix(String rd, String macAddress, String pfx, List<String> nhList,
2073               VrfEntry.EncapType encapType, int lbl, long l3vni, String gatewayMac, int addressFamily) {
2074         for (String nh : nhList) {
2075             Ipv4Address nexthop = nh != null ? new Ipv4Address(nh) : null;
2076             Long label = (long) lbl;
2077             Long afi = (long) addressFamily;
2078             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
2079                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
2080             NetworksBuilder networksBuilder = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
2081                                                 .setLabel(label).setEthtag(BgpConstants.DEFAULT_ETH_TAG)
2082                                                 .setAfi(afi);
2083             buildVpnEncapSpecificInfo(networksBuilder, encapType, label, l3vni, macAddress, gatewayMac);
2084             update(iid, networksBuilder.build());
2085         }
2086     }
2087
2088     private static void buildVpnEncapSpecificInfo(NetworksBuilder builder, VrfEntry.EncapType encapType, long label,
2089                                                   long l3vni, String macAddress, String gatewayMac) {
2090         if (encapType.equals(VrfEntry.EncapType.Mplsgre)) {
2091             builder.setLabel(label).setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLL3VPN)
2092                     .setEncapType(EncapType.GRE);
2093         } else {
2094             builder.setL3vni(l3vni).setMacaddress(macAddress).setRoutermac(gatewayMac)
2095                     .setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLEVPN).setEncapType(EncapType.VXLAN);
2096         }
2097     }
2098
2099     // TODO: add LayerType as arg - supports command
2100     public synchronized void addVrf(String rd, List<String> irts, List<String> erts, LayerType layerType) {
2101         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2102                 InstanceIdentifier.builder(Bgp.class)
2103                         .child(Vrfs.class, new VrfsKey(rd));
2104         InstanceIdentifier<Vrfs> iid = iib.build();
2105         Vrfs dto = new VrfsBuilder().setRd(rd).setImportRts(irts)
2106                 .setExportRts(erts).setLayerType(layerType).build();
2107         try {
2108             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
2109         } catch (TransactionCommitFailedException e) {
2110             LOG.error("Error adding VRF to datastore", e);
2111             throw new RuntimeException(e);
2112         }
2113     }
2114
2115     public synchronized void stopConfig() {
2116         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2117                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2118         InstanceIdentifier<ConfigServer> iid = iib.build();
2119         delete(iid);
2120     }
2121
2122     public synchronized void stopBgp() {
2123         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2124                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2125         InstanceIdentifier<AsId> iid = iib.build();
2126         delete(iid);
2127     }
2128
2129     public synchronized void delLogging() {
2130         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2131                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2132         InstanceIdentifier<Logging> iid = iib.build();
2133         delete(iid);
2134     }
2135
2136     public synchronized void delGracefulRestart() {
2137         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2138                 InstanceIdentifier.builder(Bgp.class)
2139                         .child(GracefulRestart.class);
2140         InstanceIdentifier<GracefulRestart> iid = iib.build();
2141         delete(iid);
2142     }
2143
2144     public synchronized void delNeighbor(String nbrIp) {
2145         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2146         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2147                 InstanceIdentifier.builder(Bgp.class)
2148                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2149         InstanceIdentifier<Neighbors> iid = iib.build();
2150         delete(iid);
2151     }
2152
2153     public synchronized void delUpdateSource(String nbrIp) {
2154         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2155         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2156                 InstanceIdentifier.builder(Bgp.class)
2157                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2158                         .child(UpdateSource.class);
2159         InstanceIdentifier<UpdateSource> iid = iib.build();
2160         delete(iid);
2161     }
2162
2163     public synchronized void delEbgpMultihop(String nbrIp) {
2164         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2165         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2166                 InstanceIdentifier.builder(Bgp.class)
2167                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2168                         .child(EbgpMultihop.class);
2169         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2170         delete(iid);
2171     }
2172
2173     public synchronized void delAddressFamily(String nbrIp, int afi, int safi) {
2174         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2175         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2176                 InstanceIdentifier.builder(Bgp.class)
2177                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2178                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2179         InstanceIdentifier<AddressFamilies> iid = iib.build();
2180         delete(iid);
2181     }
2182
2183     public synchronized void delPrefix(String rd, String pfx, int afi) {
2184         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
2185                 InstanceIdentifier.builder(Bgp.class)
2186                         .child(Networks.class, new NetworksKey(pfx, rd));
2187         InstanceIdentifier<Networks> iid = iib.build();
2188         delete(iid);
2189     }
2190
2191     public synchronized void delVrf(String rd) {
2192         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2193                 InstanceIdentifier.builder(Bgp.class)
2194                         .child(Vrfs.class, new VrfsKey(rd));
2195         InstanceIdentifier<Vrfs> iid = iib.build();
2196         delete(iid);
2197     }
2198
2199     public synchronized void setMultipathStatus(af_afi afi, af_safi safi, boolean enable) {
2200         long lafi = afi.getValue();
2201         long lsafi = safi.getValue();
2202
2203         InstanceIdentifier.InstanceIdentifierBuilder<Multipath> iib =
2204                 InstanceIdentifier
2205                         .builder(Bgp.class)
2206                         .child(Multipath.class,
2207                                 new MultipathKey(Long.valueOf(afi.getValue()), Long.valueOf(safi.getValue())));
2208
2209         Multipath dto = new MultipathBuilder().setAfi(lafi).setSafi(lsafi).setMultipathEnabled(enable).build();
2210         update(iib.build(), dto);
2211     }
2212
2213     public synchronized void multipaths(String rd, int maxpath) {
2214         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
2215                 InstanceIdentifier
2216                         .builder(Bgp.class)
2217                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
2218
2219         VrfMaxpath dto = new VrfMaxpathBuilder().setRd(rd).setMaxpaths(maxpath).build();
2220         update(iib.build(), dto);
2221     }
2222
2223     static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
2224
2225     /*
2226     * Remove Stale Marked Routes after timer expiry.
2227     */
2228     class RouteCleanup implements Callable<Integer> {
2229
2230         @Override
2231         public Integer call() {
2232             totalCleared = 0;
2233             try {
2234                 if (staledFibEntriesMap.isEmpty()) {
2235                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
2236                 } else {
2237                     for (String rd : staledFibEntriesMap.keySet()) {
2238                         if (Thread.interrupted()) {
2239                             return 0;
2240                         }
2241                         Map<String, Long> map = staledFibEntriesMap.get(rd);
2242                         if (map != null) {
2243                             for (String key : map.keySet()) {
2244                                 if (Thread.interrupted()) {
2245                                     return 0;
2246                                 }
2247                                 String prefix = extractPrefix(key);
2248                                 String nextHop = extractNextHop(key);
2249                                 totalCleared++;
2250                                 LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}, nextHop:{}",
2251                                         rd, prefix, nextHop);
2252                                 fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix, nextHop);
2253                             }
2254                         }
2255                     }
2256                 }
2257             } finally {
2258                 staledFibEntriesMap.clear();
2259             }
2260             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
2261             return 0;
2262         }
2263     }
2264
2265     /*
2266      * BGP restart scenario, ODL-BGP manager was/is running.
2267      * On re-sync notification, Get a copy of FIB database.
2268      */
2269     public static void createStaleFibMap() {
2270         totalStaledCount = 0;
2271         try {
2272             /*
2273             * at the time Stale FIB creation, Wait till all PENDING write transaction
2274              * to complete (or)wait for max timeout value of STALE_FIB_WAIT Seconds.
2275              */
2276             int retry = STALE_FIB_WAIT;
2277             while ((BgpUtil.getGetPendingWrTransaction() != 0) && (retry > 0)) {
2278                 Thread.sleep(1000);
2279                 retry--;
2280                 if (retry == 0) {
2281                     LOG.error("TimeOut occured {} seconds, in waiting stale fibDSWriter create", STALE_FIB_WAIT);
2282                 }
2283             }
2284             staledFibEntriesMap.clear();
2285             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2286             DataBroker db = BgpUtil.getBroker();
2287             if (db == null) {
2288                 LOG.error("Couldn't find BgpUtil dataBroker while creating createStaleFibMap");
2289                 return;
2290             }
2291
2292             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(BgpUtil.getBroker(),
2293                     LogicalDatastoreType.CONFIGURATION, id);
2294             if (fibEntries.isPresent()) {
2295                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2296                 for (VrfTables vrfTable : staleVrfTables) {
2297                     Map<String, Long> staleFibEntMap = new HashMap<>();
2298                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2299                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2300                             //Stale marking and cleanup is only meant for the routes learned through BGP.
2301                             continue;
2302                         }
2303                         if (Thread.interrupted()) {
2304                             break;
2305                         }
2306                         totalStaledCount++;
2307                         //Create MAP from staleVrfTables.
2308                         vrfEntry.getRoutePaths()
2309                                 .forEach(
2310                                     routePath -> staleFibEntMap.put(
2311                                             appendNextHopToPrefix(vrfEntry.getDestPrefix(),
2312                                                     routePath.getNexthopAddress()), routePath.getLabel()));
2313                     }
2314                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), staleFibEntMap);
2315                 }
2316             } else {
2317                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
2318             }
2319         } catch (InterruptedException | ReadFailedException e) {
2320             LOG.error("createStaleFibMap:: error ", e);
2321         }
2322         LOG.error("created {} staled entries ", totalStaledCount);
2323     }
2324
2325     //map<rd, map<prefix/len:nexthop, label>>
2326     public static Map<String, Map<String, Long>> getStaledFibEntriesMap() {
2327         return staledFibEntriesMap;
2328     }
2329
2330     //TODO: below function is for testing purpose with cli
2331     public static void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
2332         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
2333         fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nexthop);
2334     }
2335
2336     public boolean isBgpConnected() {
2337         return bgpRouter.isBgpConnected();
2338     }
2339
2340     public long getLastConnectedTS() {
2341         return bgpRouter.getLastConnectedTS();
2342     }
2343
2344     public long getConnectTS() {
2345         return bgpRouter.getConnectTS();
2346     }
2347
2348     public long getStartTS() {
2349         return bgpRouter.getStartTS();
2350     }
2351
2352     public static int getTotalStaledCount() {
2353         return totalStaledCount;
2354     }
2355
2356     public static int getTotalCleared() {
2357         return totalCleared;
2358     }
2359
2360     public Timer getBgpCountersTimer() {
2361         return bgpCountersTimer;
2362     }
2363
2364     public BgpCounters getBgpCounters() {
2365         return bgpCounters;
2366     }
2367
2368     public void setBgpCountersTimer(Timer timer) {
2369         bgpCountersTimer = timer;
2370     }
2371
2372     public void setBgpAlarmsTimer(Timer timer) {
2373         bgpAlarmsTimer = timer;
2374     }
2375
2376     public void startBgpCountersTask() {
2377         if (getBgpCounters() == null) {
2378             bgpCounters = new BgpCounters(bgpConfigurationManager.getBgpSdncMipIp());
2379             setBgpCountersTimer(new Timer(true));
2380             getBgpCountersTimer().scheduleAtFixedRate(bgpCounters, 0, 120 * 1000);
2381             LOG.info("Bgp Counters task scheduled for every two minutes.");
2382
2383             bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
2384         }
2385     }
2386
2387     public void stopBgpCountersTask() {
2388         Timer timer = getBgpCountersTimer();
2389         if (getBgpCounters() != null) {
2390             timer.cancel();
2391             setBgpCountersTimer(null);
2392             bgpCounters = null;
2393         }
2394     }
2395
2396     public void startBgpAlarmsTask() {
2397         if (getBgpAlarms() == null) {
2398             bgpAlarms = new BgpAlarms(this);
2399             setBgpAlarmsTimer(new Timer(true));
2400             getBgpAlarmsTimer().scheduleAtFixedRate(bgpAlarms, 0, 60 * 1000);
2401             LOG.info("Bgp Alarms task scheduled for every minute.");
2402         }
2403     }
2404
2405     public void stopBgpAlarmsTask() {
2406         Timer timer = getBgpAlarmsTimer();
2407         if (getBgpAlarms() != null) {
2408             timer.cancel();
2409             setBgpAlarmsTimer(null);
2410             bgpAlarms = null;
2411         }
2412     }
2413
2414     public Timer getBgpAlarmsTimer() {
2415         return bgpAlarmsTimer;
2416     }
2417
2418     public BgpAlarms getBgpAlarms() {
2419         return bgpAlarms;
2420     }
2421
2422     private static String appendNextHopToPrefix(String prefix, String nextHop) {
2423         return prefix + ":" + nextHop;
2424     }
2425
2426     private static String extractPrefix(String prefixNextHop) {
2427         return prefixNextHop.split(":")[0];
2428     }
2429
2430     private static String extractNextHop(String prefixNextHop) {
2431         return prefixNextHop.split(":")[1];
2432     }
2433 }