vpnservice/bgpmanager/bgpmanager-impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import com.google.common.base.Optional;
  11 import io.netty.util.concurrent.GlobalEventExecutor;
  12 import java.io.BufferedReader;
  13 import java.io.File;
  14 import java.io.FileNotFoundException;
  15 import java.io.FileReader;
  16 import java.io.IOException;
  17 import java.lang.reflect.Constructor;
  18 import java.lang.reflect.InvocationTargetException;
  19 import java.net.InetAddress;
  20 import java.net.NetworkInterface;
  21 import java.net.SocketException;
  22 import java.util.Collections;
  23 import java.util.Enumeration;
  24 import java.util.HashMap;
  25 import java.util.Iterator;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Timer;
  29 import java.util.TimerTask;
  30 import java.util.concurrent.Callable;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.CountDownLatch;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.concurrent.ExecutorService;
  35 import java.util.concurrent.Executors;
  36 import java.util.concurrent.Future;
  37 import java.util.concurrent.ScheduledExecutorService;
  38 import java.util.concurrent.TimeoutException;
  39 import java.util.concurrent.atomic.AtomicBoolean;
  40 import java.util.concurrent.atomic.AtomicInteger;
  41 import javax.annotation.Nullable;
  42
  43
  44 import org.apache.thrift.TException;
  45 import org.opendaylight.controller.config.api.osgi.WaitingServiceTracker;
  46 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  47 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  48 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  49 import org.opendaylight.controller.md.sal.common.api.clustering.CandidateAlreadyRegisteredException;
  50 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipService;
  51 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  52 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  53 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  54 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  55 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  56 import org.opendaylight.genius.utils.batching.DefaultBatchHandler;
  57 import org.opendaylight.genius.utils.clustering.EntityOwnerUtils;
  58 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.AddressFamiliesReactor;
  59 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.AsIdReactor;
  60 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.BgpReactor;
  61 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.ConfigServerReactor;
  62 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.EbgpMultihopReactor;
  63 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.GracefulRestartReactor;
  64 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.LoggingReactor;
  65 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.MultipathReactor;
  66 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.NeighborsReactor;
  67 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.NetworksReactor;
  68 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.RouteCleanup;
  69 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.UpdateSourceReactor;
  70 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.VrfMaxpathReactor;
  71 import org.opendaylight.netvirt.bgpmanager.BgpConfigurationManager.VrfsReactor;
  72 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  73 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  74 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  75 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  76 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  77 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  78 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  79 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  80 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  81 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  82 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  83 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  84 import org.opendaylight.netvirt.bgpmanager.thrift.gen.protocol_type;
  85 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  86 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  87 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  88 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  89 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
  90 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EncapType;
  91 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.LayerType;
  92 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.TcpMd5SignaturePasswordType;
  93 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
  94 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
  96 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
  98 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
 100 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Multipath;
 102 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathKey;
 104 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Neighbors;
 105 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsKey;
 107 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Networks;
 108 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksKey;
 110 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpath;
 111 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathKey;
 113 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Vrfs;
 114 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsKey;
 116 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamilies;
 117 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesBuilder;
 118 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesKey;
 119 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihop;
 120 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihopBuilder;
 121 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSource;
 122 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSourceBuilder;
 123 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.TcpSecurityOption;
 124 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOption;
 125 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOptionBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
 127 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 132 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 133 import org.opendaylight.yangtools.yang.binding.DataObject;
 134 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 135 import org.osgi.framework.BundleContext;
 136 import org.slf4j.Logger;
 137 import org.slf4j.LoggerFactory;
 138
 139
 140
 141 public class BgpConfigurationManager {
 142     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 143     private static DataBroker dataBroker;
 144     private static FibDSWriter fibDSWriter;
 145     public static IBgpManager bgpManager;
 146     private final BundleContext bundleContext;
 147     private static Bgp config;
 148     private static BgpRouter bgpRouter;
 149     private static BgpThriftService updateServer;
 150     private BgpCounters bgpCounters;
 151     private BgpAlarms bgpAlarms;
 152     private Timer bgpCountersTimer;
 153     private Timer bgpAlarmsTimer;
 154     private static final String DEF_LOGFILE = "/var/log/bgp_debug.log";
 155     private static final String DEF_LOGLEVEL = "errors";
 156     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 157     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 158     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 159     private static final String DEF_UPORT = "6644";
 160     private static final String DEF_CHOST = "127.0.0.1";
 161     private static final String DEF_CPORT = "7644";
 162     private static final String DEF_SDNC_BGP_MIP = "127.0.0.1";
 163     private static final String DEF_BGP_SDNC_MIP = "127.0.0.1";
 164     private static final String SDNC_BGP_MIP = "vpnservice.bgp.thrift.bgp.mip";
 165     private static final String BGP_SDNC_MIP = "vpnservice.bgp.thrift.sdnc.mip";
 166     private static final String CLUSTER_CONF_FILE = "/cluster/etc/cluster.conf";
 167     private static final Timer IP_ACTIVATION_CHECK_TIMER = new Timer();
 168     private static final int STALE_FIB_WAIT = 60;
 169     private static final int RESTART_DEFAULT_GR = 90;
 170     private long staleStartTime = 0;
 171     private long staleEndTime = 0;
 172     private long cfgReplayStartTime = 0;
 173     private long cfgReplayEndTime = 0;
 174     private long staleCleanupTime = 0;
 175     private static final int DS_RETRY_COOUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
 176     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
 177
 178     public String getBgpSdncMipIp() {
 179         return getProperty(BGP_SDNC_MIP, DEF_BGP_SDNC_MIP);
 180     }
 181
 182     public long getStaleCleanupTime() {
 183         return staleCleanupTime;
 184     }
 185
 186     public void setStaleCleanupTime(long staleCleanupTime) {
 187         this.staleCleanupTime = staleCleanupTime;
 188     }
 189
 190     public long getCfgReplayEndTime() {
 191         return cfgReplayEndTime;
 192     }
 193
 194     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 195         this.cfgReplayEndTime = cfgReplayEndTime;
 196     }
 197
 198     public long getCfgReplayStartTime() {
 199         return cfgReplayStartTime;
 200     }
 201
 202     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 203         this.cfgReplayStartTime = cfgReplayStartTime;
 204     }
 205
 206     public long getStaleEndTime() {
 207         return staleEndTime;
 208     }
 209
 210     public void setStaleEndTime(long staleEndTime) {
 211         this.staleEndTime = staleEndTime;
 212     }
 213
 214     public long getStaleStartTime() {
 215         return staleStartTime;
 216     }
 217
 218     public void setStaleStartTime(long staleStartTime) {
 219         this.staleStartTime = staleStartTime;
 220     }
 221
 222
 223     // to have stale FIB map (RD, Prefix)
 224     //  number of seconds wait for route sync-up between ODL and BGP
 225     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 226
 227     static String odlThriftIp = "127.0.0.1";
 228     static String bgpThriftIp = "127.0.0.1";
 229     private static String cHostStartup;
 230     private static String cPortStartup;
 231     private static CountDownLatch initer = new CountDownLatch(1);
 232     //static IITMProvider itmProvider;
 233     //map<rd, map<prefix/len:nexthop, label>>
 234     private static Map<String, Map<String, Long>> staledFibEntriesMap = new ConcurrentHashMap<>();
 235
 236     static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 237     static final String BGP_ENTITY_NAME = "bgp";
 238
 239     static int totalStaledCount = 0;
 240     static int totalCleared = 0;
 241     static int totalExternalRoutes = 0;
 242
 243     private static final Class[] REACTORS = {
 244         ConfigServerReactor.class, AsIdReactor.class,
 245         GracefulRestartReactor.class, LoggingReactor.class,
 246         NeighborsReactor.class, UpdateSourceReactor.class,
 247         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 248         NetworksReactor.class, VrfsReactor.class, BgpReactor.class,
 249         MultipathReactor.class, VrfMaxpathReactor.class
 250     };
 251
 252     private ListenerRegistration<DataChangeListener>[] registrations;
 253
 254     final BgpConfigurationManager bgpConfigurationManager;
 255
 256     public BgpConfigurationManager(final DataBroker dataBroker,
 257             final EntityOwnershipService entityOwnershipService,
 258             final FibDSWriter fibDSWriter,
 259             final BundleContext bundleContext)
 260             throws InterruptedException, ExecutionException, TimeoutException {
 261         BgpConfigurationManager.dataBroker = dataBroker;
 262         BgpConfigurationManager.fibDSWriter = fibDSWriter;
 263         this.bundleContext = bundleContext;
 264         String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
 265         cHostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 266         cPortStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 267         LOG.info("UpdateServer at localhost:" + updatePort + " ConfigServer at "
 268                 + cHostStartup + ":" + cPortStartup);
 269         VtyshCli.setHostAddr(cHostStartup);
 270         ClearBgpCli.setHostAddr(cHostStartup);
 271         setEntityOwnershipService(entityOwnershipService);
 272         bgpRouter = BgpRouter.getInstance();
 273         odlThriftIp = getProperty(SDNC_BGP_MIP, DEF_SDNC_BGP_MIP);
 274         bgpThriftIp = getProperty(BGP_SDNC_MIP, DEF_BGP_SDNC_MIP);
 275         registerCallbacks();
 276
 277         LOG.info("BGP Configuration manager initialized");
 278         initer.countDown();
 279
 280         bgpConfigurationManager = this;
 281         BgpUtil.batchSize = BgpUtil.BATCH_SIZE;
 282         if (Integer.getInteger("batch.size") != null) {
 283             BgpUtil.batchSize = Integer.getInteger("batch.size");
 284         }
 285         BgpUtil.batchInterval = BgpUtil.PERIODICITY;
 286         if (Integer.getInteger("batch.wait.time") != null) {
 287             BgpUtil.batchInterval = Integer.getInteger("batch.wait.time");
 288         }
 289         BgpUtil.registerWithBatchManager(
 290                 new DefaultBatchHandler(dataBroker, LogicalDatastoreType.CONFIGURATION, BgpUtil.batchSize,
 291                         BgpUtil.batchInterval));
 292
 293         GlobalEventExecutor.INSTANCE.execute(() -> {
 294             final WaitingServiceTracker<IBgpManager> tracker = WaitingServiceTracker.create(
 295                     IBgpManager.class, bundleContext);
 296             bgpManager = tracker.waitForService(WaitingServiceTracker.FIVE_MINUTES);
 297             updateServer = new BgpThriftService(Integer.parseInt(updatePort), bgpManager, fibDSWriter);
 298             updateServer.start();
 299             LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 300         });
 301     }
 302
 303     private Object createListener(Class<?> cls) {
 304         Constructor<?> ctor;
 305         Object obj = null;
 306
 307         try {
 308             ctor = cls.getConstructor(BgpConfigurationManager.class);
 309             obj = ctor.newInstance(this);
 310         } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException
 311                 e) {
 312             LOG.error("Failed to create listener object", e);
 313         }
 314         return obj;
 315     }
 316
 317     private void registerCallbacks() {
 318         String emsg = "Failed to register listener";
 319         registrations = new ListenerRegistration[REACTORS.length];
 320         InstanceIdentifier<?> iid = InstanceIdentifier.create(Bgp.class);
 321         for (Class reactor : REACTORS) {
 322             Object obj = createListener(reactor);
 323             AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 324             dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 325         }
 326     }
 327
 328     public void close() {
 329         if (updateServer != null) {
 330             updateServer.stop();
 331         }
 332         LOG.info("{} close", getClass().getSimpleName());
 333     }
 334
 335     private boolean configExists() throws ReadFailedException {
 336         InstanceIdentifier.InstanceIdentifierBuilder<Bgp> iib =
 337                 InstanceIdentifier.builder(Bgp.class);
 338         InstanceIdentifier<Bgp> iid = iib.build();
 339         return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 340                 iid).isPresent();
 341     }
 342
 343     private String getProperty(String var, String def) {
 344         String property = bundleContext.getProperty(var);
 345         return (property == null ? def : property);
 346     }
 347
 348     boolean ignoreClusterDcnEventForFollower() {
 349         return !EntityOwnerUtils.amIEntityOwner(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME);
 350     }
 351
 352     public Bgp get() {
 353         config = getConfig();
 354         return config;
 355     }
 356
 357     public void setEntityOwnershipService(final EntityOwnershipService entityOwnershipService) {
 358         try {
 359             EntityOwnerUtils.registerEntityCandidateForOwnerShip(entityOwnershipService,
 360                 BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME, ownershipChange -> {
 361                     LOG.trace("entity owner change event fired");
 362                     if (ownershipChange.hasOwner() && ownershipChange.isOwner()) {
 363                         LOG.trace("This PL is the Owner");
 364                         activateMIP();
 365                         bgpRestarted();
 366                     } else {
 367                         LOG.info("Not owner: hasOwner: {}, isOwner: {}", ownershipChange.hasOwner(),
 368                                 ownershipChange.isOwner());
 369                     }
 370                 });
 371         } catch (CandidateAlreadyRegisteredException e) {
 372             LOG.error("failed to register bgp entity", e);
 373         }
 374     }
 375
 376     private static final String ADD_WARN =
 377             "Config store updated; undo with Delete if needed.";
 378     private static final String DEL_WARN =
 379             "Config store updated; undo with Add if needed.";
 380     private static final String UPD_WARN =
 381             "Update operation not supported; Config store updated;"
 382                     + " restore with another Update if needed.";
 383
 384     public class ConfigServerReactor
 385             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 386             implements AutoCloseable, ClusteredDataTreeChangeListener<ConfigServer> {
 387         private static final String YANG_OBJ = "config-server ";
 388
 389         public ConfigServerReactor() {
 390             super(ConfigServer.class, ConfigServerReactor.class);
 391         }
 392
 393         @Override
 394         protected synchronized void add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 395             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 396             if (ignoreClusterDcnEventForFollower()) {
 397                 return;
 398             }
 399
 400             try {
 401                 initer.await();
 402             } catch (InterruptedException e) {
 403                 // Ignored
 404             }
 405             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 406             synchronized (BgpConfigurationManager.this) {
 407                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 408                         val.getPort().intValue());
 409                 if (!res) {
 410                     LOG.error(YANG_OBJ + "Add failed; " + ADD_WARN);
 411                 }
 412             }
 413         }
 414
 415         @Override
 416         protected ConfigServerReactor getDataTreeChangeListener() {
 417             return ConfigServerReactor.this;
 418         }
 419
 420         @Override
 421         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 422             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 423         }
 424
 425         @Override
 426         protected synchronized void remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 427             LOG.trace("received bgp disconnect");
 428             if (ignoreClusterDcnEventForFollower()) {
 429                 return;
 430             }
 431             synchronized (BgpConfigurationManager.this) {
 432                 bgpRouter.disconnect();
 433             }
 434         }
 435
 436         @Override
 437         protected void update(InstanceIdentifier<ConfigServer> iid,
 438                 ConfigServer oldval, ConfigServer newval) {
 439             LOG.trace("received bgp Connection update");
 440             if (ignoreClusterDcnEventForFollower()) {
 441                 return;
 442             }
 443             LOG.error(YANG_OBJ + UPD_WARN);
 444         }
 445     }
 446
 447     private BgpRouter getClient(String yangObj) {
 448         if (bgpRouter == null || !bgpRouter.isBgpConnected()) {
 449             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 450             return null;
 451         }
 452         return bgpRouter;
 453     }
 454
 455     public class AsIdReactor
 456             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 457             implements AutoCloseable, ClusteredDataTreeChangeListener<AsId> {
 458
 459         private static final String YANG_OBJ = "as-id ";
 460
 461         public AsIdReactor() {
 462             super(AsId.class, AsIdReactor.class);
 463         }
 464
 465         @Override
 466         protected synchronized void add(InstanceIdentifier<AsId> iid, AsId val) {
 467             LOG.error("received bgp add asid {}", val);
 468             if (ignoreClusterDcnEventForFollower()) {
 469                 return;
 470             }
 471             LOG.debug("received add router config asNum {}", val.getLocalAs());
 472             synchronized (BgpConfigurationManager.this) {
 473                 BgpRouter br = getClient(YANG_OBJ);
 474                 if (br == null) {
 475                     LOG.error("{} Unable to process add for asNum {}; {}", YANG_OBJ, val.getLocalAs(),
 476                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 477                     return;
 478                 }
 479                 if (isIpAvailable(odlThriftIp)) {
 480                     bgpRestarted();
 481                 } else {
 482                     IP_ACTIVATION_CHECK_TIMER.scheduleAtFixedRate(new TimerTask() {
 483                         @Override
 484                         public void run() {
 485                             if (isIpAvailable(odlThriftIp)) {
 486                                 bgpRestarted();
 487                                 IP_ACTIVATION_CHECK_TIMER.cancel();
 488                             } else {
 489                                 LOG.trace("waiting for odlThriftIP: {} to be present", odlThriftIp);
 490                             }
 491                         }
 492                     }, 10000L, 10000L);
 493                 }
 494                 if (getBgpCounters() == null) {
 495                     startBgpCountersTask();
 496                 }
 497                 if (getBgpAlarms() == null) {
 498                     startBgpAlarmsTask();
 499                 }
 500             }
 501         }
 502
 503         @Override
 504         protected AsIdReactor getDataTreeChangeListener() {
 505             return AsIdReactor.this;
 506         }
 507
 508         @Override
 509         protected InstanceIdentifier<AsId> getWildCardPath() {
 510             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 511         }
 512
 513         @Override
 514         protected synchronized void remove(InstanceIdentifier<AsId> iid, AsId val) {
 515             LOG.error("received delete router config asNum {}", val.getLocalAs());
 516             if (ignoreClusterDcnEventForFollower()) {
 517                 return;
 518             }
 519             synchronized (BgpConfigurationManager.this) {
 520                 long asNum = val.getLocalAs();
 521                 BgpRouter br = getClient(YANG_OBJ);
 522                 if (br == null) {
 523                     LOG.error("{} Unable to process remove for asNum {}; {}", YANG_OBJ, asNum,
 524                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 525                     return;
 526                 }
 527                 try {
 528                     br.stopBgp(asNum);
 529                 } catch (TException | BgpRouterException e) {
 530                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 531                 }
 532                 if (getBgpCounters() != null) {
 533                     stopBgpCountersTask();
 534                 }
 535                 if (getBgpAlarms() != null) {
 536                     stopBgpAlarmsTask();
 537                 }
 538                 Bgp conf = getConfig();
 539                 if (conf == null) {
 540                     LOG.error("Config Null while removing the as-id");
 541                     return;
 542                 }
 543                 LOG.debug("Removing external routes from FIB");
 544                 deleteExternalFibRoutes();
 545                 List<Neighbors> nbrs = conf.getNeighbors();
 546                 if (nbrs != null && nbrs.size() > 0) {
 547                     LOG.error("Tring to remove the as-id when neighbor config is already present");
 548                     for (Neighbors nbr : nbrs) {
 549                         LOG.debug("Removing Neighbor {} from Data store", nbr.getAddress().getValue());
 550                         delNeighbor(nbr.getAddress().getValue());
 551                     }
 552                 }
 553             }
 554         }
 555
 556         @Override
 557         protected void update(InstanceIdentifier<AsId> iid,
 558                 AsId oldval, AsId newval) {
 559             if (ignoreClusterDcnEventForFollower()) {
 560                 return;
 561             }
 562             LOG.error(YANG_OBJ + UPD_WARN);
 563         }
 564     }
 565
 566     public class GracefulRestartReactor
 567             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 568             implements AutoCloseable, ClusteredDataTreeChangeListener<GracefulRestart> {
 569
 570         private static final String YANG_OBJ = "graceful-restart ";
 571
 572         public GracefulRestartReactor() {
 573             super(GracefulRestart.class, GracefulRestartReactor.class);
 574         }
 575
 576         @Override
 577         protected synchronized void add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 578             if (ignoreClusterDcnEventForFollower()) {
 579                 return;
 580             }
 581             synchronized (BgpConfigurationManager.this) {
 582                 int stalePathTime = val.getStalepathTime().intValue();
 583                 BgpRouter br = getClient(YANG_OBJ);
 584                 if (br == null) {
 585                     LOG.error("{} Unable to add stale-path time {}; {}", YANG_OBJ, stalePathTime,
 586                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 587                     return;
 588                 }
 589                 try {
 590                     br.addGracefulRestart(stalePathTime);
 591                 } catch (TException | BgpRouterException e) {
 592                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 593                 }
 594             }
 595         }
 596
 597         @Override
 598         protected GracefulRestartReactor getDataTreeChangeListener() {
 599             return GracefulRestartReactor.this;
 600         }
 601
 602         @Override
 603         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 604             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 605         }
 606
 607         @Override
 608         protected synchronized void remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 609             if (ignoreClusterDcnEventForFollower()) {
 610                 return;
 611             }
 612             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 613             synchronized (BgpConfigurationManager.this) {
 614                 BgpRouter br = getClient(YANG_OBJ);
 615                 if (br == null) {
 616                     LOG.error("{} Unable to delete stale-path time; {}", YANG_OBJ,
 617                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 618                     return;
 619                 }
 620                 try {
 621                     br.delGracefulRestart();
 622                 } catch (TException | BgpRouterException e) {
 623                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 624                 }
 625             }
 626         }
 627
 628         @Override
 629         protected void update(InstanceIdentifier<GracefulRestart> iid,
 630                 GracefulRestart oldval, GracefulRestart newval) {
 631             if (ignoreClusterDcnEventForFollower()) {
 632                 return;
 633             }
 634             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 635             synchronized (BgpConfigurationManager.this) {
 636                 int stalePathTime = newval.getStalepathTime().intValue();
 637                 BgpRouter br = getClient(YANG_OBJ);
 638                 if (br == null) {
 639                     LOG.error("{} Unable to update stale-path time to {}; {}", YANG_OBJ, stalePathTime,
 640                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 641                     return;
 642                 }
 643                 try {
 644                     br.addGracefulRestart(stalePathTime);
 645                 } catch (TException | BgpRouterException e) {
 646                     LOG.error("{} update received exception; {}", YANG_OBJ, ADD_WARN, e);
 647                 }
 648             }
 649         }
 650     }
 651
 652     public class LoggingReactor
 653             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 654             implements AutoCloseable, ClusteredDataTreeChangeListener<Logging> {
 655
 656         private static final String YANG_OBJ = "logging ";
 657
 658         public LoggingReactor() {
 659             super(Logging.class, LoggingReactor.class);
 660         }
 661
 662         @Override
 663         protected synchronized void add(InstanceIdentifier<Logging> iid, Logging val) {
 664             if (ignoreClusterDcnEventForFollower()) {
 665                 return;
 666             }
 667             synchronized (BgpConfigurationManager.this) {
 668                 BgpRouter br = getClient(YANG_OBJ);
 669                 if (br == null) {
 670                     LOG.error("{} Unable to add logging for qbgp; {}", YANG_OBJ,
 671                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 672                     return;
 673                 }
 674                 try {
 675                     br.setLogging(val.getFile(), val.getLevel());
 676                 } catch (TException | BgpRouterException e) {
 677                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 678                 }
 679             }
 680         }
 681
 682         @Override
 683         protected LoggingReactor getDataTreeChangeListener() {
 684             return LoggingReactor.this;
 685         }
 686
 687         @Override
 688         protected InstanceIdentifier<Logging> getWildCardPath() {
 689             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 690         }
 691
 692         @Override
 693         protected synchronized void remove(InstanceIdentifier<Logging> iid, Logging val) {
 694             if (ignoreClusterDcnEventForFollower()) {
 695                 return;
 696             }
 697             LOG.debug("received remove Logging config val {}", val.getLevel());
 698             synchronized (BgpConfigurationManager.this) {
 699                 BgpRouter br = getClient(YANG_OBJ);
 700                 if (br == null) {
 701                     LOG.error("{} Unable to remove logging for qbgp; {}", YANG_OBJ,
 702                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 703                     return;
 704                 }
 705                 try {
 706                     br.setLogging(DEF_LOGFILE, DEF_LOGLEVEL);
 707                 } catch (TException | BgpRouterException e) {
 708                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 709                 }
 710             }
 711         }
 712
 713         @Override
 714         protected void update(InstanceIdentifier<Logging> iid,
 715                 Logging oldval, Logging newval) {
 716             if (ignoreClusterDcnEventForFollower()) {
 717                 return;
 718             }
 719             synchronized (BgpConfigurationManager.this) {
 720                 BgpRouter br = getClient(YANG_OBJ);
 721                 if (br == null) {
 722                     LOG.error("{} Unable to update logging for qbgp; {}", YANG_OBJ,
 723                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 724                     return;
 725                 }
 726                 try {
 727                     br.setLogging(newval.getFile(), newval.getLevel());
 728                 } catch (TException | BgpRouterException e) {
 729                     LOG.error("{} newval received exception; {}", YANG_OBJ, ADD_WARN, e);
 730                 }
 731             }
 732         }
 733     }
 734
 735     public class NeighborsReactor
 736             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 737             implements AutoCloseable, ClusteredDataTreeChangeListener<Neighbors> {
 738
 739         private static final String YANG_OBJ = "neighbors ";
 740
 741         public NeighborsReactor() {
 742             super(Neighbors.class, NeighborsReactor.class);
 743         }
 744
 745         @Override
 746         protected synchronized void add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 747             if (ignoreClusterDcnEventForFollower()) {
 748                 return;
 749             }
 750             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 751             synchronized (BgpConfigurationManager.this) {
 752                 String peerIp = val.getAddress().getValue();
 753                 long as = val.getRemoteAs();
 754                 final String md5Secret = extractMd5Secret(val);
 755                 BgpRouter br = getClient(YANG_OBJ);
 756                 if (br == null) {
 757                     LOG.error("{} Unable to process add for peer {} as {}; {}", YANG_OBJ, peerIp, as,
 758                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 759                     return;
 760                 }
 761                 try {
 762                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 763                     br.addNeighbor(peerIp, as, md5Secret);
 764
 765                 } catch (TException | BgpRouterException e) {
 766                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 767                 }
 768             }
 769         }
 770
 771         @Override
 772         protected NeighborsReactor getDataTreeChangeListener() {
 773             return NeighborsReactor.this;
 774         }
 775
 776         @Override
 777         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 778             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class);
 779         }
 780
 781         @Override
 782         protected synchronized void remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 783             if (ignoreClusterDcnEventForFollower()) {
 784                 return;
 785             }
 786             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 787             synchronized (BgpConfigurationManager.this) {
 788                 String peerIp = val.getAddress().getValue();
 789                 BgpRouter br = getClient(YANG_OBJ);
 790                 if (br == null) {
 791                     LOG.error("{} Unable to process remove for peer {}; {}", YANG_OBJ, peerIp,
 792                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 793                     return;
 794                 }
 795                 try {
 796                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 797                     br.delNeighbor(peerIp);
 798                 } catch (TException | BgpRouterException e) {
 799                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 800                 }
 801             }
 802         }
 803
 804         @Override
 805         protected void update(InstanceIdentifier<Neighbors> iid,
 806                 Neighbors oldval, Neighbors newval) {
 807             if (ignoreClusterDcnEventForFollower()) {
 808                 return;
 809             }
 810             //purposefully nothing to do.
 811         }
 812     }
 813
 814     public class EbgpMultihopReactor
 815             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 816             implements AutoCloseable, ClusteredDataTreeChangeListener<EbgpMultihop> {
 817
 818         private static final String YANG_OBJ = "ebgp-multihop ";
 819
 820         public EbgpMultihopReactor() {
 821             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 822         }
 823
 824         @Override
 825         protected synchronized void add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 826             if (ignoreClusterDcnEventForFollower()) {
 827                 return;
 828             }
 829             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 830             synchronized (BgpConfigurationManager.this) {
 831                 String peerIp = val.getPeerIp().getValue();
 832                 BgpRouter br = getClient(YANG_OBJ);
 833                 if (br == null) {
 834                     LOG.error("{} Unable to process add for peer {}; {}", YANG_OBJ, peerIp,
 835                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 836                     return;
 837                 }
 838                 try {
 839                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 840                 } catch (TException | BgpRouterException e) {
 841                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 842                 }
 843             }
 844         }
 845
 846         @Override
 847         protected EbgpMultihopReactor getDataTreeChangeListener() {
 848             return EbgpMultihopReactor.this;
 849         }
 850
 851         @Override
 852         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
 853             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(EbgpMultihop.class);
 854         }
 855
 856         @Override
 857         protected synchronized void remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 858             if (ignoreClusterDcnEventForFollower()) {
 859                 return;
 860             }
 861             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 862             synchronized (BgpConfigurationManager.this) {
 863                 String peerIp = val.getPeerIp().getValue();
 864                 BgpRouter br = getClient(YANG_OBJ);
 865                 if (br == null) {
 866                     LOG.error("{} Unable to process remove for peer {}; {}", YANG_OBJ, peerIp,
 867                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 868                     return;
 869                 }
 870                 try {
 871                     br.delEbgpMultihop(peerIp);
 872                 } catch (TException | BgpRouterException e) {
 873                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 874                 }
 875             }
 876         }
 877
 878         @Override
 879         protected void update(InstanceIdentifier<EbgpMultihop> iid,
 880                 EbgpMultihop oldval, EbgpMultihop newval) {
 881             if (ignoreClusterDcnEventForFollower()) {
 882                 return;
 883             }
 884             LOG.error(YANG_OBJ + UPD_WARN);
 885         }
 886     }
 887
 888     public class UpdateSourceReactor
 889             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
 890             implements AutoCloseable, ClusteredDataTreeChangeListener<UpdateSource> {
 891
 892         private static final String YANG_OBJ = "update-source ";
 893
 894         public UpdateSourceReactor() {
 895             super(UpdateSource.class, UpdateSourceReactor.class);
 896         }
 897
 898         @Override
 899         protected synchronized void add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 900             if (ignoreClusterDcnEventForFollower()) {
 901                 return;
 902             }
 903             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
 904             synchronized (BgpConfigurationManager.this) {
 905                 String peerIp = val.getPeerIp().getValue();
 906                 BgpRouter br = getClient(YANG_OBJ);
 907                 if (br == null) {
 908                     LOG.error("{} Unable to process add for peer {}; {}", YANG_OBJ, peerIp,
 909                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 910                     return;
 911                 }
 912                 try {
 913                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
 914                 } catch (TException | BgpRouterException e) {
 915                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 916                 }
 917             }
 918         }
 919
 920         @Override
 921         protected UpdateSourceReactor getDataTreeChangeListener() {
 922             return UpdateSourceReactor.this;
 923         }
 924
 925         @Override
 926         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
 927             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(UpdateSource.class);
 928         }
 929
 930         @Override
 931         protected synchronized void remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 932             if (ignoreClusterDcnEventForFollower()) {
 933                 return;
 934             }
 935             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
 936             synchronized (BgpConfigurationManager.this) {
 937                 String peerIp = val.getPeerIp().getValue();
 938                 BgpRouter br = getClient(YANG_OBJ);
 939                 if (br == null) {
 940                     LOG.error("{} Unable to process remove for peer {}; {}", YANG_OBJ, peerIp,
 941                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 942                     return;
 943                 }
 944                 try {
 945                     br.delUpdateSource(peerIp);
 946                 } catch (TException | BgpRouterException e) {
 947                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 948                 }
 949             }
 950         }
 951
 952         @Override
 953         protected void update(InstanceIdentifier<UpdateSource> iid,
 954                 UpdateSource oldval, UpdateSource newval) {
 955             if (ignoreClusterDcnEventForFollower()) {
 956                 return;
 957             }
 958             LOG.error(YANG_OBJ + UPD_WARN);
 959         }
 960     }
 961
 962     public class AddressFamiliesReactor
 963             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
 964             implements AutoCloseable, ClusteredDataTreeChangeListener<AddressFamilies> {
 965
 966         private static final String YANG_OBJ = "address-families ";
 967
 968         public AddressFamiliesReactor() {
 969             super(AddressFamilies.class, AddressFamiliesReactor.class);
 970         }
 971
 972         @Override
 973         protected synchronized void add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
 974             if (ignoreClusterDcnEventForFollower()) {
 975                 return;
 976             }
 977             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
 978             synchronized (BgpConfigurationManager.this) {
 979                 String peerIp = val.getPeerIp().getValue();
 980                 BgpRouter br = getClient(YANG_OBJ);
 981                 if (br == null) {
 982                     LOG.error("{} Unable to process add for peer {}; {}", YANG_OBJ, peerIp,
 983                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 984                     return;
 985                 }
 986                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
 987                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
 988                 try {
 989                     br.addAddressFamily(peerIp, afi, safi);
 990                 } catch (TException | BgpRouterException e) {
 991                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 992                 }
 993             }
 994         }
 995
 996         @Override
 997         protected AddressFamiliesReactor getDataTreeChangeListener() {
 998             return AddressFamiliesReactor.this;
 999         }
1000
1001         @Override
1002         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
1003             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(AddressFamilies.class);
1004         }
1005
1006         @Override
1007         protected synchronized void remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1008             if (ignoreClusterDcnEventForFollower()) {
1009                 return;
1010             }
1011             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
1012             synchronized (BgpConfigurationManager.this) {
1013                 String peerIp = val.getPeerIp().getValue();
1014                 BgpRouter br = getClient(YANG_OBJ);
1015                 if (br == null) {
1016                     LOG.error("{} Unable to process remove for peer {}; {}", YANG_OBJ, peerIp,
1017                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1018                     return;
1019                 }
1020                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1021                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1022                 try {
1023                     br.delAddressFamily(peerIp, afi, safi);
1024                 } catch (TException | BgpRouterException e) {
1025                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1026                 }
1027             }
1028         }
1029
1030         @Override
1031         protected void update(InstanceIdentifier<AddressFamilies> iid,
1032                 AddressFamilies oldval, AddressFamilies newval) {
1033             if (ignoreClusterDcnEventForFollower()) {
1034                 return;
1035             }
1036             LOG.error(YANG_OBJ + UPD_WARN);
1037         }
1038     }
1039
1040     public class NetworksReactor
1041             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
1042             implements AutoCloseable, ClusteredDataTreeChangeListener<Networks> {
1043
1044         private static final String YANG_OBJ = "networks ";
1045
1046         public NetworksReactor() {
1047             super(Networks.class, NetworksReactor.class);
1048         }
1049
1050         @Override
1051         public NetworksReactor getDataTreeChangeListener() {
1052             return NetworksReactor.this;
1053         }
1054
1055         @Override
1056         protected synchronized void add(InstanceIdentifier<Networks> iid, Networks val) {
1057             if (ignoreClusterDcnEventForFollower()) {
1058                 return;
1059             }
1060             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1061             synchronized (BgpConfigurationManager.this) {
1062                 String rd = val.getRd();
1063                 String pfxlen = val.getPrefixLen();
1064                 String nh = val.getNexthop().getValue();
1065                 BgpRouter br = getClient(YANG_OBJ);
1066                 if (br == null) {
1067                     LOG.error("{} Unable to process add for rd {} prefix {} nexthop {}; {}", YANG_OBJ, rd, pfxlen, nh,
1068                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1069                     return;
1070                 }
1071                 Long label = val.getLabel();
1072                 int lbl = (label == null) ? qbgpConstants.LBL_NO_LABEL
1073                         : label.intValue();
1074                 int l3vni = (val.getL3vni() == null) ? qbgpConstants.LBL_NO_LABEL
1075                         : val.getL3vni().intValue();
1076
1077                 BgpControlPlaneType protocolType = val.getBgpControlPlaneType();
1078                 int ethernetTag = val.getEthtag().intValue();
1079                 String esi = val.getEsi();
1080                 String macaddress = val.getMacaddress();
1081                 EncapType encapType = val.getEncapType();
1082                 String routerMac = val.getRoutermac();
1083
1084                 try {
1085                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, BgpUtil.convertToThriftProtocolType(protocolType),
1086                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1087                 } catch (TException | BgpRouterException e) {
1088                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1089                 }
1090             }
1091         }
1092
1093         @Override
1094         protected InstanceIdentifier<Networks> getWildCardPath() {
1095             return InstanceIdentifier.create(Bgp.class).child(Networks.class);
1096         }
1097
1098         @Override
1099         protected synchronized void remove(InstanceIdentifier<Networks> iid, Networks val) {
1100             if (ignoreClusterDcnEventForFollower()) {
1101                 return;
1102             }
1103             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1104             synchronized (BgpConfigurationManager.this) {
1105                 String rd = val.getRd();
1106                 String pfxlen = val.getPrefixLen();
1107                 BgpRouter br = getClient(YANG_OBJ);
1108                 if (br == null) {
1109                     LOG.error("{} Unable to process remove for rd {} prefix {}; {}", YANG_OBJ, rd, pfxlen,
1110                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1111                     return;
1112                 }
1113                 Long label = val.getLabel();
1114                 int lbl = (label == null) ? 0 : label.intValue();
1115                 if (rd == null && lbl > 0) {
1116                     //LU prefix is being deleted.
1117                     rd = Integer.toString(lbl);
1118                 }
1119                 try {
1120                     br.delPrefix(rd, pfxlen);
1121                 } catch (TException | BgpRouterException e) {
1122                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1123                 }
1124             }
1125         }
1126
1127         @Override
1128         protected void update(final InstanceIdentifier<Networks> iid,
1129                 final Networks oldval, final Networks newval) {
1130             if (ignoreClusterDcnEventForFollower()) {
1131                 return;
1132             }
1133             if (oldval.equals(newval)) {
1134                 //Update: OLD and New values are same, no need to trigger remove/add.
1135                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1136                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1137                 return;
1138             }
1139             LOG.debug("received update networks config val {}", newval.getPrefixLen());
1140             remove(iid, oldval);
1141             timer.schedule(new TimerTask() {
1142                 @Override
1143                 public void run() {
1144                     add(iid, newval);
1145                 }
1146             }, Integer.getInteger("bgp.nexthop.update.delay.in.secs", 5) * 1000);
1147         }
1148     }
1149
1150     static Timer timer = new Timer();
1151
1152     public class VrfsReactor
1153             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1154             implements AutoCloseable, ClusteredDataTreeChangeListener<Vrfs> {
1155
1156         private static final String YANG_OBJ = "vrfs ";
1157
1158         public VrfsReactor() {
1159             super(Vrfs.class, VrfsReactor.class);
1160         }
1161
1162         @Override
1163         protected synchronized void add(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1164             if (ignoreClusterDcnEventForFollower()) {
1165                 return;
1166             }
1167             LOG.debug("received add Vrfs config val {}", val.getRd());
1168             synchronized (BgpConfigurationManager.this) {
1169                 String rd = val.getRd();
1170                 BgpRouter br = getClient(YANG_OBJ);
1171                 if (br == null) {
1172                     LOG.error("{} Unable to process add for rd {}; {}", YANG_OBJ, rd,
1173                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1174                     return;
1175                 }
1176                 try {
1177                     br.addVrf(val.getLayerType(), rd, val.getImportRts(),
1178                             val.getExportRts());
1179                 } catch (TException | BgpRouterException e) {
1180                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1181                 }
1182             }
1183         }
1184
1185         @Override
1186         protected VrfsReactor getDataTreeChangeListener() {
1187             return VrfsReactor.this;
1188         }
1189
1190         @Override
1191         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1192             return InstanceIdentifier.create(Bgp.class).child(Vrfs.class);
1193         }
1194
1195         @Override
1196         protected synchronized void remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1197             if (ignoreClusterDcnEventForFollower()) {
1198                 return;
1199             }
1200             LOG.debug("received remove Vrfs config val {}", val.getRd());
1201             synchronized (BgpConfigurationManager.this) {
1202                 String rd = val.getRd();
1203                 BgpRouter br = getClient(YANG_OBJ);
1204                 if (br == null) {
1205                     LOG.error("{} Unable to process remove for rd {}; {}", YANG_OBJ, rd,
1206                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1207                     return;
1208                 }
1209                 try {
1210                     br.delVrf(rd);
1211                 } catch (TException | BgpRouterException e) {
1212                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1213                 }
1214             }
1215         }
1216
1217         @Override
1218         protected void update(InstanceIdentifier<Vrfs> iid,
1219                 Vrfs oldval, Vrfs newval) {
1220             if (ignoreClusterDcnEventForFollower()) {
1221                 return;
1222             }
1223             LOG.debug("VRFS: Update getting triggered for VRFS rd {}", oldval.getRd());
1224             LOG.error(YANG_OBJ + UPD_WARN);
1225         }
1226     }
1227
1228     Future lastCleanupJob;
1229     Future lastReplayJobFt = null;
1230
1231     protected void activateMIP() {
1232         try {
1233             LOG.trace("BgpReactor: Executing MIP Activate command");
1234             Process processBgp = Runtime.getRuntime().exec("cluster ip -a sdnc_bgp_mip");
1235             Process processOs = Runtime.getRuntime().exec("cluster ip -a sdnc_os_mip");
1236             LOG.trace("bgpMIP Activated");
1237
1238         } catch (IOException io) {
1239             LOG.error("IO Exception got while activating mip: {}", io.getMessage());
1240         }
1241     }
1242
1243     AtomicBoolean started = new AtomicBoolean(false);
1244
1245     public class BgpReactor
1246             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1247             implements AutoCloseable, ClusteredDataTreeChangeListener<Bgp> {
1248
1249         private static final String YANG_OBJ = "Bgp ";
1250
1251         public BgpReactor() {
1252             super(Bgp.class, BgpReactor.class);
1253         }
1254
1255
1256         @Override
1257         protected synchronized void add(InstanceIdentifier<Bgp> iid, Bgp val) {
1258             LOG.error("received add Bgp config");
1259
1260             try {
1261                 initer.await();
1262             } catch (InterruptedException e) {
1263                 // Ignored
1264             }
1265             synchronized (BgpConfigurationManager.this) {
1266                 config = val;
1267                 if (ignoreClusterDcnEventForFollower()) {
1268                     return;
1269                 }
1270                 activateMIP();
1271             }
1272         }
1273
1274         @Override
1275         protected BgpReactor getDataTreeChangeListener() {
1276             return BgpReactor.this;
1277         }
1278
1279         @Override
1280         protected InstanceIdentifier<Bgp> getWildCardPath() {
1281             return InstanceIdentifier.create(Bgp.class);
1282         }
1283
1284         @Override
1285         protected synchronized void remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1286             if (ignoreClusterDcnEventForFollower()) {
1287                 return;
1288             }
1289             LOG.debug("received remove Bgp config");
1290             synchronized (BgpConfigurationManager.this) {
1291                 config = null;
1292             }
1293         }
1294
1295         @Override
1296         protected void update(InstanceIdentifier<Bgp> iid,
1297                 Bgp oldval, Bgp newval) {
1298             if (ignoreClusterDcnEventForFollower()) {
1299                 return;
1300             }
1301             synchronized (BgpConfigurationManager.this) {
1302                 config = newval;
1303             }
1304         }
1305     }
1306
1307     @SuppressWarnings("deprecation")
1308     public class MultipathReactor
1309             extends AsyncDataTreeChangeListenerBase<Multipath, MultipathReactor>
1310             implements AutoCloseable, ClusteredDataTreeChangeListener<Multipath> {
1311
1312         private static final String YANG_OBJ = "multipath ";
1313
1314         public MultipathReactor() {
1315             super(Multipath.class, MultipathReactor.class);
1316         }
1317
1318
1319         @Override
1320         protected MultipathReactor getDataTreeChangeListener() {
1321             return MultipathReactor.this;
1322         }
1323
1324         @Override
1325         protected InstanceIdentifier<Multipath> getWildCardPath() {
1326             return InstanceIdentifier.create(Bgp.class).child(Multipath.class);
1327         }
1328
1329         @Override
1330         protected synchronized void remove(InstanceIdentifier<Multipath> iid, Multipath val) {
1331             ExecutorService executor = Executors.newSingleThreadExecutor();
1332             executor.submit(new MultipathStatusChange(val));
1333             executor.shutdown();
1334         }
1335
1336         @Override
1337         protected void update(InstanceIdentifier<Multipath> iid, Multipath oldval, Multipath newval) {
1338             ExecutorService executor = Executors.newSingleThreadExecutor();
1339             executor.submit(new MultipathStatusChange(newval));
1340             executor.shutdown();
1341         }
1342
1343         @Override
1344         protected void add(InstanceIdentifier<Multipath> key, Multipath dataObjectModification) {
1345             ExecutorService executor = Executors.newSingleThreadExecutor();
1346             executor.submit(new MultipathStatusChange(dataObjectModification));
1347             executor.shutdown();
1348         }
1349
1350         class MultipathStatusChange implements Callable<Void> {
1351
1352             Multipath multipath;
1353
1354             MultipathStatusChange(Multipath multipath) {
1355                 this.multipath = multipath;
1356             }
1357
1358             @Override
1359             public Void call() throws Exception {
1360                 if (!ignoreClusterDcnEventForFollower()) {
1361                     synchronized (BgpConfigurationManager.this) {
1362
1363                         BgpRouter br = getClient(YANG_OBJ);
1364
1365                         if (br != null) {
1366                             af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1367                             af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1368
1369                             try {
1370                                 if (multipath.isMultipathEnabled()) {
1371                                     br.enableMultipath(afi, safi);
1372                                 } else {
1373                                     br.disableMultipath(afi, safi);
1374                                 }
1375                             } catch (TException | BgpRouterException e) {
1376                                 LOG.error(YANG_OBJ + " received exception: \"" + e + "\"");
1377                             }
1378                         }
1379                     }
1380                 }
1381                 return null;
1382             }
1383
1384         }
1385
1386         @Override
1387         public void close() {
1388             super.close();
1389         }
1390     }
1391
1392     @SuppressWarnings("deprecation")
1393     public class VrfMaxpathReactor
1394             extends AsyncDataTreeChangeListenerBase<VrfMaxpath, VrfMaxpathReactor>
1395             implements AutoCloseable, ClusteredDataTreeChangeListener<VrfMaxpath> {
1396
1397         private static final String YANG_OBJ = "vrfMaxpath ";
1398
1399         public VrfMaxpathReactor() {
1400             super(VrfMaxpath.class, VrfMaxpathReactor.class);
1401         }
1402
1403
1404         @Override
1405         protected VrfMaxpathReactor getDataTreeChangeListener() {
1406             return VrfMaxpathReactor.this;
1407         }
1408
1409         @Override
1410         protected InstanceIdentifier<VrfMaxpath> getWildCardPath() {
1411             return InstanceIdentifier.create(Bgp.class).child(VrfMaxpath.class);
1412         }
1413
1414         class VrfMaxPathConfigurator implements Callable<Void> {
1415
1416             VrfMaxpath vrfMaxpathVal;
1417
1418             VrfMaxPathConfigurator(VrfMaxpath vrfMaxPathVal) {
1419                 this.vrfMaxpathVal = vrfMaxPathVal;
1420             }
1421
1422             @Override
1423             public Void call() throws Exception {
1424                 if (!ignoreClusterDcnEventForFollower()) {
1425                     synchronized (BgpConfigurationManager.this) {
1426                         BgpRouter br = getClient(YANG_OBJ);
1427                         if (br != null) {
1428                             try {
1429                                 br.multipaths(vrfMaxpathVal.getRd(), vrfMaxpathVal.getMaxpaths());
1430                                 LOG.debug("Maxpath for vrf: " + vrfMaxpathVal.getRd() + " : is "
1431                                         + vrfMaxpathVal.getMaxpaths());
1432                             } catch (TException | BgpRouterException e) {
1433                                 LOG.error(YANG_OBJ
1434                                         + " received exception: \"" + e + "\"");
1435                             }
1436                         }
1437                     }
1438                 }
1439                 return null;
1440             }
1441         }
1442
1443         @Override
1444         protected synchronized void remove(InstanceIdentifier<VrfMaxpath> iid, VrfMaxpath vrfMaxPathVal) {
1445             ExecutorService executor = Executors.newSingleThreadExecutor();
1446             executor.submit(new VrfMaxPathConfigurator(vrfMaxPathVal));
1447             executor.shutdown();
1448         }
1449
1450         @Override
1451         protected void update(InstanceIdentifier<VrfMaxpath> iid,
1452                               VrfMaxpath oldval, VrfMaxpath newval) {
1453             if (oldval.getMaxpaths() != newval.getMaxpaths()) {
1454                 ExecutorService executor = Executors.newSingleThreadExecutor();
1455                 executor.submit(new VrfMaxPathConfigurator(newval));
1456                 executor.shutdown();
1457             }
1458         }
1459
1460         @Override
1461         protected void add(InstanceIdentifier<VrfMaxpath> instanceIdentifier, VrfMaxpath vrfMaxpathVal) {
1462             ExecutorService executor = Executors.newSingleThreadExecutor();
1463             executor.submit(new VrfMaxPathConfigurator(vrfMaxpathVal));
1464             executor.shutdown();
1465         }
1466
1467         @Override
1468         public void close() {
1469             super.close();
1470         }
1471     }
1472
1473     public String readThriftIpForCommunication(String mipAddr) {
1474         File file = new File(CLUSTER_CONF_FILE);
1475         if (!file.exists()) {
1476             return DEF_CHOST;
1477         }
1478
1479         try (BufferedReader br = new BufferedReader(new FileReader(file))) {
1480             String line;
1481             while ((line = br.readLine()) != null) {
1482                 if (line.contains(mipAddr)) {
1483                     line = line.trim();
1484                     return line.substring(line.lastIndexOf(" ") + 1);
1485                 }
1486             }
1487         } catch (FileNotFoundException e) {
1488             return DEF_CHOST;
1489         } catch (IOException e) {
1490             LOG.error("Error reading {}", CLUSTER_CONF_FILE, e);
1491         }
1492         return DEF_CHOST;
1493     }
1494
1495     public boolean isIpAvailable(String odlip) {
1496
1497         try {
1498             if (odlip != null) {
1499                 if ("127.0.0.1".equals(odlip)) {
1500                     return true;
1501                 }
1502                 Enumeration networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces();
1503                 while (networkInterfaceEnumeration.hasMoreElements()) {
1504                     NetworkInterface networkInterface = (NetworkInterface) networkInterfaceEnumeration.nextElement();
1505                     Enumeration inetAddressEnumeration = networkInterface.getInetAddresses();
1506                     while (inetAddressEnumeration.hasMoreElements()) {
1507                         InetAddress inetAddress = (InetAddress) inetAddressEnumeration.nextElement();
1508                         if (odlip.equals(inetAddress.getHostAddress())) {
1509                             return true;
1510                         }
1511                     }
1512                 }
1513             }
1514         } catch (SocketException e) {
1515             // Ignored?
1516         }
1517         return false;
1518     }
1519
1520     public static long getStalePathtime(int defValue, AsId asId) {
1521         long spt = 0;
1522         try {
1523             spt = getConfig().getGracefulRestart().getStalepathTime();
1524         } catch (NullPointerException e) {
1525             try {
1526                 spt = asId.getStalepathTime();
1527                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1528             } catch (NullPointerException ignore) {
1529                 LOG.trace("BGP AS id is not set using graceful");
1530                 spt = defValue;
1531             }
1532         }
1533         if (spt == 0) {
1534             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1535             spt = defValue;
1536         }
1537         return spt;
1538     }
1539
1540     public synchronized void bgpRestarted() {
1541         /*
1542          * If there a thread which in the process of stale cleanup, cancel it
1543          * and start a new thread (to avoid processing same again).
1544          */
1545         if (previousReplayJobInProgress()) {
1546             cancelPreviousReplayJob();
1547         }
1548         Runnable task = () -> {
1549             try {
1550                 LOG.info("running bgp replay task ");
1551                 if (get() == null) {
1552                     String host = getConfigHost();
1553                     int port = getConfigPort();
1554                     LOG.info("connecting  to bgp host {} ", host);
1555
1556                     boolean res = bgpRouter.connect(host, port);
1557                     LOG.info("no config to push in bgp replay task ");
1558                     return;
1559                 }
1560                 setStaleStartTime(System.currentTimeMillis());
1561                 LOG.info("started creating stale fibDSWriter  map ");
1562                 createStaleFibMap();
1563                 setStaleEndTime(System.currentTimeMillis());
1564                 LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
1565                 LOG.info("started bgp config replay ");
1566                 setCfgReplayStartTime(System.currentTimeMillis());
1567                 try {
1568                     replay();
1569                 } catch (TimeoutException | ExecutionException e) {
1570                     LOG.error("Error while replaying routes. {}", e);
1571                 }
1572                 setCfgReplayEndTime(System.currentTimeMillis());
1573                 LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1574                 long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1575                 Thread.sleep(routeSyncTime * 1000L);
1576                 setStaleCleanupTime(routeSyncTime);
1577                 new RouteCleanup().call();
1578             } catch (InterruptedException eCancel) {
1579                 LOG.error("Stale Cleanup Task Cancelled", eCancel);
1580             }
1581         };
1582         lastReplayJobFt = executor.submit(task);
1583     }
1584
1585     private boolean previousReplayJobInProgress() {
1586         return lastReplayJobFt != null && !lastReplayJobFt.isDone();
1587     }
1588
1589     private void cancelPreviousReplayJob() {
1590         try {
1591             LOG.error("cancelling already running bgp replay task");
1592             lastReplayJobFt.cancel(true);
1593             lastReplayJobFt = null;
1594             Thread.sleep(2000);
1595         } catch (InterruptedException e) {
1596             LOG.error("Failed to cancel previous replay job ", e);
1597         }
1598     }
1599
1600     private static void doRouteSync() throws InterruptedException, TimeoutException, ExecutionException {
1601         BgpSyncHandle bsh = BgpSyncHandle.getInstance();
1602         LOG.error("Starting BGP route sync");
1603         try {
1604             bgpRouter.initRibSync(bsh);
1605         } catch (TException | BgpRouterException e) {
1606             LOG.error("Route sync aborted, exception when initializing", e);
1607             return;
1608         }
1609         while (bsh.getState() != bsh.DONE) {
1610             Routes routes = null;
1611             try {
1612                 routes = bgpRouter.doRibSync(bsh);
1613             } catch (TException | BgpRouterException e) {
1614                 LOG.error("Route sync aborted, exception when syncing", e);
1615                 return;
1616             }
1617             Iterator<Update> updates = routes.getUpdatesIterator();
1618             while (updates.hasNext()) {
1619                 Update update = updates.next();
1620                 Map<String, Map<String, Long>> staleFibRdMap = BgpConfigurationManager.getStaledFibEntriesMap();
1621                 String rd = update.getRd();
1622                 String nexthop = update.getNexthop();
1623
1624                 // TODO: decide correct label here
1625                 int label = update.getL3label();
1626
1627                 String prefix = update.getPrefix();
1628                 int plen = update.getPrefixlen();
1629
1630
1631                 // TODO: protocol type will not be available in "update"
1632                 // use "rd" to query vrf table and obtain the protocol_type. Currently using PROTOCOL_EVPN as default.
1633                 onUpdatePushRoute(
1634                         protocol_type.PROTOCOL_EVPN,
1635                         rd,
1636                         prefix,
1637                         plen,
1638                         nexthop,
1639                         update.getEthtag(),
1640                         update.getEsi(),
1641                         update.getMacaddress(),
1642                         label,
1643                         update.getRoutermac()
1644                 );
1645             }
1646         }
1647         try {
1648             LOG.error("Ending BGP route-sync");
1649             bgpRouter.endRibSync(bsh);
1650         } catch (TException | BgpRouterException e) {
1651             // Ignored?
1652         }
1653     }
1654
1655     /* onUpdatePushRoute
1656      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1657      *  - Entry compare shall include NextHop, Label.
1658      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1659      *  - If entry not found, add to FIB Config DS.
1660      *  - If entry found, but either Label/NextHop doesn't match.
1661      *      - Update FIB Config DS with modified values.
1662      *      - delete from Stale Map.
1663      */
1664
1665     public static void onUpdatePushRoute(protocol_type protocolType,
1666                                          String rd,
1667                                          String prefix,
1668                                          int plen,
1669                                          String nextHop,
1670                                          int ethtag,
1671                                          String esi,
1672                                          String macaddress,
1673                                          int label,
1674                                          String routermac)
1675             throws InterruptedException, ExecutionException, TimeoutException {
1676         boolean addroute = false;
1677         long l3vni = 0L;
1678         VrfEntry.EncapType encapType = VrfEntry.EncapType.Mplsgre;
1679         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1680             encapType = VrfEntry.EncapType.Vxlan;
1681             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = BgpUtil.getVpnInstanceOpData(dataBroker, rd);
1682             if (vpnInstanceOpDataEntry != null) {
1683                 l3vni = vpnInstanceOpDataEntry.getL3vni();
1684             } else {
1685                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1686                 return;
1687             }
1688         }
1689         if (!staledFibEntriesMap.isEmpty()) {
1690             // restart Scenario, as MAP is not empty.
1691             Map<String, Long> map = staledFibEntriesMap.get(rd);
1692             if (map != null) {
1693                 String prefixNextHop = appendNextHopToPrefix(prefix + "/" + plen, nextHop);
1694                 Long labelInStaleMap = map.get(prefixNextHop);
1695                 if (null == labelInStaleMap) {
1696                     // New Entry, which happened to be added during restart.
1697                     addroute = true;
1698                 } else {
1699                     map.remove(prefixNextHop);
1700                     if (isRouteModified(label, labelInStaleMap)) {
1701                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1702                         // Existing entry, where in Label got modified during restart
1703                         addroute = true;
1704                     }
1705                 }
1706             }
1707         } else {
1708             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1709             addroute = true;
1710         }
1711         if (addroute) {
1712             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1713             // TODO: modify addFibEntryToDS signature
1714             fibDSWriter.addFibEntryToDS(rd, macaddress, prefix + "/" + plen, Collections.singletonList(nextHop),
1715                     encapType, label, l3vni, routermac, RouteOrigin.BGP);
1716             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1717         }
1718     }
1719
1720     private static boolean isRouteModified(int label, Long labelInStaleMap) {
1721         return labelInStaleMap != null && !labelInStaleMap.equals(Long.valueOf(label));
1722     }
1723
1724     private static void replayNbrConfig(List<Neighbors> neighbors, BgpRouter br) {
1725         for (Neighbors nbr : neighbors) {
1726             try {
1727                 final String md5password = extractMd5Secret(nbr);
1728                 br.addNeighbor(nbr.getAddress().getValue(),
1729                         nbr.getRemoteAs(), md5password);
1730                 //itmProvider.buildTunnelsToDCGW(new IpAddress(nbr.getAddress().getValue().toCharArray()));
1731             } catch (TException | BgpRouterException e) {
1732                 LOG.error("Replay:addNbr() received exception", e);
1733                 continue;
1734             }
1735             EbgpMultihop en = nbr.getEbgpMultihop();
1736             if (en != null) {
1737                 try {
1738                     br.addEbgpMultihop(en.getPeerIp().getValue(),
1739                             en.getNhops().intValue());
1740                 } catch (TException | BgpRouterException e) {
1741                     LOG.error("Replay:addEBgp() received exception", e);
1742                 }
1743             }
1744             UpdateSource us = nbr.getUpdateSource();
1745             if (us != null) {
1746                 try {
1747                     br.addUpdateSource(us.getPeerIp().getValue(),
1748                             us.getSourceIp().getValue());
1749                 } catch (TException | BgpRouterException e) {
1750                     LOG.error("Replay:addUS() received exception", e);
1751                 }
1752             }
1753             List<AddressFamilies> afs = nbr.getAddressFamilies();
1754             if (afs != null) {
1755                 for (AddressFamilies af : afs) {
1756                     af_afi afi = af_afi.findByValue(af.getAfi().intValue());
1757                     af_safi safi = af_safi.findByValue(af.getSafi().intValue());
1758                     try {
1759                         br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
1760                     } catch (TException | BgpRouterException e) {
1761                         LOG.error("Replay:addAf() received exception", e);
1762                     }
1763                 }
1764             }
1765         }
1766     }
1767
1768     public static String getConfigHost() {
1769         if (config == null) {
1770             return cHostStartup;
1771         }
1772         ConfigServer ts = config.getConfigServer();
1773         return (ts == null ? cHostStartup : ts.getHost().getValue());
1774     }
1775
1776     public static int getConfigPort() {
1777         if (config == null) {
1778             return Integer.parseInt(cPortStartup);
1779         }
1780         ConfigServer ts = config.getConfigServer();
1781         return (ts == null ? Integer.parseInt(cPortStartup) :
1782                 ts.getPort().intValue());
1783     }
1784
1785     public static Bgp getConfig() {
1786         AtomicInteger bgpDSretryCount = new AtomicInteger(DS_RETRY_COOUNT);
1787         while (0 != bgpDSretryCount.decrementAndGet()) {
1788             try {
1789                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
1790                         InstanceIdentifier.create(Bgp.class)).orNull();
1791             } catch (ReadFailedException e) {
1792                 //Config DS may not be up, so sleep for 1 second and retry
1793                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
1794                 try {
1795                     Thread.sleep(WAIT_TIME_BETWEEN_EACH_TRY_MILLIS);
1796                 } catch (InterruptedException timerEx) {
1797                     LOG.debug("WAIT_TIME_BETWEEN_EACH_TRY_MILLIS, Timer got interrupted while waiting for"
1798                             + "config DS availability", timerEx);
1799                 }
1800             }
1801         }
1802         LOG.error("failed to get bgp config");
1803         return null;
1804     }
1805
1806     @SuppressWarnings("checkstyle:IllegalCatch")
1807     public synchronized void replay() throws InterruptedException, TimeoutException, ExecutionException {
1808         synchronized (bgpConfigurationManager) {
1809             String host = getConfigHost();
1810             int port = getConfigPort();
1811             LOG.error("connecting  to bgp host {} ", host);
1812
1813             boolean res = bgpRouter.connect(host, port);
1814             if (!res) {
1815                 String msg = "Cannot connect to BGP config server at " + host + ":" + port;
1816                 if (config != null) {
1817                     msg += "; Configuration Replay aborted";
1818                 }
1819                 LOG.error(msg);
1820                 return;
1821             }
1822             config = getConfig();
1823             if (config == null) {
1824                 LOG.error("bgp config is empty nothing to push to bgp");
1825                 return;
1826             }
1827             BgpRouter br = bgpRouter;
1828             AsId asId = config.getAsId();
1829             if (asId == null) {
1830                 return;
1831             }
1832             long asNum = asId.getLocalAs();
1833             IpAddress routerId = asId.getRouterId();
1834             Long spt = asId.getStalepathTime();
1835             Boolean afb = asId.isAnnounceFbit();
1836             String rid = (routerId == null) ? "" : new String(routerId.getValue());
1837             int stalepathTime = (int) getStalePathtime(RESTART_DEFAULT_GR, config.getAsId());
1838             boolean announceFbit = true;
1839             try {
1840                 br.startBgp(asNum, rid, stalepathTime, announceFbit);
1841             } catch (BgpRouterException bre) {
1842                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
1843                     doRouteSync();
1844                 } else {
1845                     LOG.error("Replay: startBgp() received exception: \""
1846                             + bre + "\"; " + ADD_WARN);
1847                 }
1848             } catch (TException e) {
1849                 //not unusual. We may have restarted & BGP is already on
1850                 LOG.error("Replay:startBgp() received exception: \"" + e + "\"");
1851             }
1852
1853             if (getBgpCounters() == null) {
1854                 startBgpCountersTask();
1855             }
1856
1857             if (getBgpAlarms() == null) {
1858                 startBgpAlarmsTask();
1859             }
1860
1861             Logging logging = config.getLogging();
1862             if (logging != null) {
1863                 try {
1864                     br.setLogging(logging.getFile(), logging.getLevel());
1865                 } catch (TException | BgpRouterException e) {
1866                     LOG.error("Replay:setLogging() received exception", e);
1867                 }
1868             }
1869
1870             GracefulRestart gracefulRestart = config.getGracefulRestart();
1871             if (gracefulRestart != null) {
1872                 try {
1873                     br.addGracefulRestart(gracefulRestart.getStalepathTime().intValue());
1874                 } catch (TException | BgpRouterException e) {
1875                     LOG.error("Replay:addGr() received exception", e);
1876                 }
1877             }
1878
1879             List<Vrfs> vrfs = config.getVrfs();
1880             if (vrfs != null) {
1881                 for (Vrfs vrf : vrfs) {
1882                     try {
1883                         br.addVrf(vrf.getLayerType(), vrf.getRd(), vrf.getImportRts(),
1884                                 vrf.getExportRts());
1885                     } catch (TException | BgpRouterException e) {
1886                         LOG.error("Replay:addVrf() received exception", e);
1887                     }
1888                 }
1889             }
1890
1891             List<Networks> ln = config.getNetworks();
1892             if (ln != null) {
1893                 for (Networks net : ln) {
1894                     String rd = net.getRd();
1895                     String pfxlen = net.getPrefixLen();
1896                     String nh = net.getNexthop().getValue();
1897                     Long label = net.getLabel();
1898                     int lbl = (label == null) ? 0 : label.intValue();
1899                     int l3vni = (net.getL3vni() == null) ? 0 : net.getL3vni().intValue();
1900                     if (rd == null && lbl > 0) {
1901                         //LU prefix is being deleted.
1902                         rd = Integer.toString(lbl);
1903                     }
1904
1905                     BgpControlPlaneType protocolType = net.getBgpControlPlaneType();
1906                     int ethernetTag = net.getEthtag().intValue();
1907                     String esi = net.getEsi();
1908                     String macaddress = net.getMacaddress();
1909                     EncapType encapType = net.getEncapType();
1910                     String routerMac = net.getRoutermac();
1911
1912                     try {
1913                         br.addPrefix(rd, pfxlen, nh, lbl, l3vni, BgpUtil.convertToThriftProtocolType(protocolType),
1914                                 ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1915                     } catch (Exception e) {
1916                         LOG.error("Replay:addPfx() received exception", e);
1917                     }
1918                 }
1919             }
1920             List<Neighbors> neighbors = config.getNeighbors();
1921             if (neighbors != null) {
1922                 LOG.error("configuring existing Neighbors present for replay total neighbors {}", neighbors.size());
1923                 replayNbrConfig(neighbors, br);
1924             } else {
1925                 LOG.error("no Neighbors present for replay config ");
1926             }
1927
1928             List<Multipath> multipaths = config.getMultipath();
1929
1930             if (multipaths != null) {
1931                 for (Multipath multipath: multipaths) {
1932                     if (multipath != null) {
1933                         af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1934                         af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1935
1936                         try {
1937                             if (multipath.isMultipathEnabled()) {
1938                                 br.enableMultipath(afi, safi);
1939                             } else {
1940                                 br.disableMultipath(afi, safi);
1941                             }
1942                         } catch (TException | BgpRouterException e) {
1943                             LOG.info("Replay:multipaths() received exception: \"" + e + "\"");
1944                         }
1945                     }
1946                 }
1947             }
1948             List<VrfMaxpath> vrfMaxpaths = config.getVrfMaxpath();
1949             if (vrfMaxpaths != null) {
1950                 for (VrfMaxpath vrfMaxpath: vrfMaxpaths) {
1951                     try {
1952                         br.multipaths(vrfMaxpath.getRd(), vrfMaxpath.getMaxpaths());
1953                     } catch (TException | BgpRouterException e) {
1954                         LOG.info("Replay:vrfMaxPath() received exception: \"" + e + "\"");
1955                     }
1956                 }
1957             }
1958         }
1959     }
1960
1961     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
1962         BgpUtil.update(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1963     }
1964
1965     private <T extends DataObject> void asyncWrite(InstanceIdentifier<T> iid, T dto) {
1966         BgpUtil.write(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1967     }
1968
1969     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
1970         BgpUtil.delete(dataBroker, LogicalDatastoreType.CONFIGURATION, iid);
1971     }
1972
1973     public void startConfig(String bgpHost, int thriftPort) {
1974         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
1975                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
1976         InstanceIdentifier<ConfigServer> iid = iib.build();
1977         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
1978         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
1979                 .setPort((long) thriftPort).build();
1980         update(iid, dto);
1981     }
1982
1983     public void startBgp(long as, String routerId, int spt, boolean fbit) {
1984         IpAddress rid = (routerId == null) ? null : new IpAddress(routerId.toCharArray());
1985         Long staleTime = (long) spt;
1986         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
1987                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
1988         InstanceIdentifier<AsId> iid = iib.build();
1989         AsId dto = new AsIdBuilder().setLocalAs(as)
1990                 .setRouterId(rid)
1991                 .setStalepathTime(staleTime)
1992                 .setAnnounceFbit(fbit).build();
1993         update(iid, dto);
1994     }
1995
1996     public void addLogging(String fileName, String logLevel) {
1997         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
1998                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
1999         InstanceIdentifier<Logging> iid = iib.build();
2000         Logging dto = new LoggingBuilder().setFile(fileName)
2001                 .setLevel(logLevel).build();
2002         update(iid, dto);
2003     }
2004
2005     public void addGracefulRestart(int staleTime) {
2006         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2007                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
2008         InstanceIdentifier<GracefulRestart> iid = iib.build();
2009         GracefulRestart dto = new GracefulRestartBuilder()
2010                 .setStalepathTime((long) staleTime).build();
2011         update(iid, dto);
2012     }
2013
2014     public void addNeighbor(
2015             String nbrIp, long remoteAs, @Nullable final TcpMd5SignaturePasswordType md5Secret) {
2016         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2017         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2018                 InstanceIdentifier.builder(Bgp.class)
2019                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2020         InstanceIdentifier<Neighbors> iid = iib.build();
2021         TcpSecurityOption tcpSecOption = null;
2022         if (md5Secret != null) {
2023             tcpSecOption = new TcpMd5SignatureOptionBuilder().setTcpMd5SignaturePassword(md5Secret).build();
2024         } // else let tcpSecOption be null
2025         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
2026                 .setRemoteAs(remoteAs).setTcpSecurityOption(tcpSecOption).build();
2027         update(iid, dto);
2028     } // public addNeighbor(nbrIp, remoteAs, md5Secret)
2029
2030     public void addUpdateSource(String nbrIp, String srcIp) {
2031         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2032         Ipv4Address srcAddr = new Ipv4Address(srcIp);
2033         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2034                 InstanceIdentifier.builder(Bgp.class)
2035                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2036                         .child(UpdateSource.class);
2037         InstanceIdentifier<UpdateSource> iid = iib.build();
2038         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
2039                 .setSourceIp(srcAddr).build();
2040         update(iid, dto);
2041     }
2042
2043     public void addEbgpMultihop(String nbrIp, int hops) {
2044         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2045         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2046                 InstanceIdentifier.builder(Bgp.class)
2047                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2048                         .child(EbgpMultihop.class);
2049         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2050         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
2051                 .setNhops((long) hops).build();
2052         update(iid, dto);
2053     }
2054
2055     public void addAddressFamily(String nbrIp, int afi, int safi) {
2056         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2057         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2058                 InstanceIdentifier.builder(Bgp.class)
2059                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2060                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2061         InstanceIdentifier<AddressFamilies> iid = iib.build();
2062         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
2063                 .setAfi((long) afi).setSafi((long) safi).build();
2064         update(iid, dto);
2065     }
2066
2067     public void addPrefix(String rd, String macAddress, String pfx, List<String> nhList,
2068               VrfEntry.EncapType encapType, long lbl, long l3vni, long l2vni, String gatewayMac, int addressFamily) {
2069         for (String nh : nhList) {
2070             Ipv4Address nexthop = nh != null ? new Ipv4Address(nh) : null;
2071             Long label = lbl;
2072             Long afi = (long) addressFamily;
2073             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
2074                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
2075             NetworksBuilder networksBuilder = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
2076                                                 .setLabel(label).setEthtag(BgpConstants.DEFAULT_ETH_TAG)
2077                                                 .setAfi(afi);
2078             buildVpnEncapSpecificInfo(networksBuilder, encapType, label, l3vni, l2vni, macAddress, gatewayMac);
2079             update(iid, networksBuilder.build());
2080         }
2081     }
2082
2083     private static void buildVpnEncapSpecificInfo(NetworksBuilder builder, VrfEntry.EncapType encapType, long label,
2084                                                   long l3vni, long l2vni, String macAddress, String gatewayMac) {
2085         if (encapType.equals(VrfEntry.EncapType.Mplsgre)) {
2086             builder.setLabel(label).setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLL3VPN)
2087                     .setEncapType(EncapType.GRE);
2088         } else {
2089             builder.setL3vni(l3vni).setL2vni(l2vni).setMacaddress(macAddress).setRoutermac(gatewayMac)
2090                     .setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLEVPN).setEncapType(EncapType.VXLAN);
2091         }
2092     }
2093
2094     // TODO: add LayerType as arg - supports command
2095     public void addVrf(String rd, List<String> irts, List<String> erts, LayerType layerType) {
2096         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2097                 InstanceIdentifier.builder(Bgp.class)
2098                         .child(Vrfs.class, new VrfsKey(rd));
2099         InstanceIdentifier<Vrfs> iid = iib.build();
2100         Vrfs dto = new VrfsBuilder().setRd(rd).setImportRts(irts)
2101                 .setExportRts(erts).setLayerType(layerType).build();
2102         try {
2103             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
2104         } catch (TransactionCommitFailedException e) {
2105             LOG.error("Error adding VRF to datastore", e);
2106             throw new RuntimeException(e);
2107         }
2108     }
2109
2110     public void stopConfig() {
2111         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2112                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2113         InstanceIdentifier<ConfigServer> iid = iib.build();
2114         delete(iid);
2115     }
2116
2117     public void stopBgp() {
2118         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2119                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2120         InstanceIdentifier<AsId> iid = iib.build();
2121         delete(iid);
2122     }
2123
2124     public void delLogging() {
2125         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2126                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2127         InstanceIdentifier<Logging> iid = iib.build();
2128         delete(iid);
2129     }
2130
2131     public void delGracefulRestart() {
2132         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2133                 InstanceIdentifier.builder(Bgp.class)
2134                         .child(GracefulRestart.class);
2135         InstanceIdentifier<GracefulRestart> iid = iib.build();
2136         delete(iid);
2137     }
2138
2139     public void delNeighbor(String nbrIp) {
2140         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2141         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2142                 InstanceIdentifier.builder(Bgp.class)
2143                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2144         InstanceIdentifier<Neighbors> iid = iib.build();
2145         delete(iid);
2146     }
2147
2148     public void delUpdateSource(String nbrIp) {
2149         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2150         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2151                 InstanceIdentifier.builder(Bgp.class)
2152                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2153                         .child(UpdateSource.class);
2154         InstanceIdentifier<UpdateSource> iid = iib.build();
2155         delete(iid);
2156     }
2157
2158     public void delEbgpMultihop(String nbrIp) {
2159         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2160         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2161                 InstanceIdentifier.builder(Bgp.class)
2162                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2163                         .child(EbgpMultihop.class);
2164         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2165         delete(iid);
2166     }
2167
2168     public void delAddressFamily(String nbrIp, int afi, int safi) {
2169         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2170         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2171                 InstanceIdentifier.builder(Bgp.class)
2172                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2173                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2174         InstanceIdentifier<AddressFamilies> iid = iib.build();
2175         delete(iid);
2176     }
2177
2178     public void delPrefix(String rd, String pfx, int afi) {
2179         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
2180                 InstanceIdentifier.builder(Bgp.class)
2181                         .child(Networks.class, new NetworksKey(pfx, rd));
2182         InstanceIdentifier<Networks> iid = iib.build();
2183         delete(iid);
2184     }
2185
2186     public void delVrf(String rd) {
2187         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2188                 InstanceIdentifier.builder(Bgp.class)
2189                         .child(Vrfs.class, new VrfsKey(rd));
2190         InstanceIdentifier<Vrfs> iid = iib.build();
2191         delete(iid);
2192     }
2193
2194     public void setMultipathStatus(af_afi afi, af_safi safi, boolean enable) {
2195         long lafi = afi.getValue();
2196         long lsafi = safi.getValue();
2197
2198         InstanceIdentifier.InstanceIdentifierBuilder<Multipath> iib =
2199                 InstanceIdentifier
2200                         .builder(Bgp.class)
2201                         .child(Multipath.class,
2202                                 new MultipathKey(Long.valueOf(afi.getValue()), Long.valueOf(safi.getValue())));
2203
2204         Multipath dto = new MultipathBuilder().setAfi(lafi).setSafi(lsafi).setMultipathEnabled(enable).build();
2205         update(iib.build(), dto);
2206     }
2207
2208     public void multipaths(String rd, int maxpath) {
2209         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
2210                 InstanceIdentifier
2211                         .builder(Bgp.class)
2212                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
2213
2214         VrfMaxpath dto = new VrfMaxpathBuilder().setRd(rd).setMaxpaths(maxpath).build();
2215         update(iib.build(), dto);
2216     }
2217
2218     static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
2219
2220     /*
2221     * Remove Stale Marked Routes after timer expiry.
2222     */
2223     class RouteCleanup implements Callable<Integer> {
2224
2225         @Override
2226         public Integer call() {
2227             totalCleared = 0;
2228             try {
2229                 if (staledFibEntriesMap.isEmpty()) {
2230                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
2231                 } else {
2232                     for (String rd : staledFibEntriesMap.keySet()) {
2233                         if (Thread.interrupted()) {
2234                             return 0;
2235                         }
2236                         Map<String, Long> map = staledFibEntriesMap.get(rd);
2237                         if (map != null) {
2238                             for (String key : map.keySet()) {
2239                                 if (Thread.interrupted()) {
2240                                     return 0;
2241                                 }
2242                                 String prefix = extractPrefix(key);
2243                                 String nextHop = extractNextHop(key);
2244                                 totalCleared++;
2245                                 LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}, nextHop:{}",
2246                                         rd, prefix, nextHop);
2247                                 fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix, nextHop);
2248                             }
2249                         }
2250                     }
2251                 }
2252             } finally {
2253                 staledFibEntriesMap.clear();
2254             }
2255             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
2256             return 0;
2257         }
2258     }
2259
2260     /*
2261      * BGP restart scenario, ODL-BGP manager was/is running.
2262      * On re-sync notification, Get a copy of FIB database.
2263      */
2264     public static void createStaleFibMap() {
2265         totalStaledCount = 0;
2266         try {
2267             /*
2268             * at the time Stale FIB creation, Wait till all PENDING write transaction
2269              * to complete (or)wait for max timeout value of STALE_FIB_WAIT Seconds.
2270              */
2271             int retry = STALE_FIB_WAIT;
2272             while ((BgpUtil.getGetPendingWrTransaction() != 0) && (retry > 0)) {
2273                 Thread.sleep(1000);
2274                 retry--;
2275                 if (retry == 0) {
2276                     LOG.error("TimeOut occured {} seconds, in waiting stale fibDSWriter create", STALE_FIB_WAIT);
2277                 }
2278             }
2279             staledFibEntriesMap.clear();
2280             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2281             DataBroker db = BgpUtil.getBroker();
2282             if (db == null) {
2283                 LOG.error("Couldn't find BgpUtil dataBroker while creating createStaleFibMap");
2284                 return;
2285             }
2286
2287             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(BgpUtil.getBroker(),
2288                     LogicalDatastoreType.CONFIGURATION, id);
2289             if (fibEntries.isPresent()) {
2290                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2291                 for (VrfTables vrfTable : staleVrfTables) {
2292                     Map<String, Long> staleFibEntMap = new HashMap<>();
2293                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2294                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2295                             //Stale marking and cleanup is only meant for the routes learned through BGP.
2296                             continue;
2297                         }
2298                         if (Thread.interrupted()) {
2299                             break;
2300                         }
2301                         totalStaledCount++;
2302                         //Create MAP from staleVrfTables.
2303                         vrfEntry.getRoutePaths()
2304                                 .forEach(
2305                                     routePath -> staleFibEntMap.put(
2306                                             appendNextHopToPrefix(vrfEntry.getDestPrefix(),
2307                                                     routePath.getNexthopAddress()), routePath.getLabel()));
2308                     }
2309                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), staleFibEntMap);
2310                 }
2311             } else {
2312                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
2313             }
2314         } catch (InterruptedException | ReadFailedException e) {
2315             LOG.error("createStaleFibMap:: error ", e);
2316         }
2317         LOG.error("created {} staled entries ", totalStaledCount);
2318     }
2319
2320     /*
2321      * BGP config remove scenario, Need to remove all the
2322      * external routes from FIB.
2323      */
2324     public static void deleteExternalFibRoutes() {
2325         totalExternalRoutes = 0;
2326         try {
2327             /*
2328             * at the time FIB route deletion, Wait till all PENDING write transaction
2329              * to complete (or)wait for max timeout value of STALE_FIB_WAIT Seconds.
2330              */
2331             int retry = STALE_FIB_WAIT;
2332             String rd;
2333             while ((BgpUtil.getGetPendingWrTransaction() != 0) && (retry > 0)) {
2334                 Thread.sleep(1000);
2335                 retry--;
2336                 if (retry == 0) {
2337                     LOG.error("TimeOut occured {} seconds, while deleting external routes", STALE_FIB_WAIT);
2338                 }
2339             }
2340             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2341             DataBroker db = BgpUtil.getBroker();
2342             if (db == null) {
2343                 LOG.error("Couldn't find BgpUtil dataBroker while deleting external routes");
2344                 return;
2345             }
2346
2347             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(BgpUtil.getBroker(),
2348                     LogicalDatastoreType.CONFIGURATION, id);
2349             if (fibEntries.isPresent()) {
2350                 if (fibEntries.get().getVrfTables() == null) {
2351                     LOG.error("deleteExternalFibRoutes::getVrfTables is null");
2352                     return;
2353                 }
2354                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2355                 for (VrfTables vrfTable : staleVrfTables) {
2356                     rd = vrfTable.getRouteDistinguisher();
2357                     if (vrfTable.getVrfEntry() == null) {
2358                         LOG.error("deleteExternalFibRoutes::getVrfEntry is null");
2359                         continue;
2360                     }
2361                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2362                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2363                             //route cleanup is only meant for the routes learned through BGP.
2364                             continue;
2365                         }
2366                         totalExternalRoutes++;
2367                         fibDSWriter.removeFibEntryFromDS(rd, vrfEntry.getDestPrefix());
2368                     }
2369                 }
2370             } else {
2371                 LOG.error("deleteExternalFibRoutes:: FIBentries.class is not present");
2372             }
2373         } catch (InterruptedException | ReadFailedException e) {
2374             LOG.error("deleteExternalFibRoutes:: error ", e);
2375         }
2376         LOG.debug("deleted {} fib entries ", totalExternalRoutes);
2377     }
2378
2379     //map<rd, map<prefix/len:nexthop, label>>
2380     public static Map<String, Map<String, Long>> getStaledFibEntriesMap() {
2381         return staledFibEntriesMap;
2382     }
2383
2384     //TODO: below function is for testing purpose with cli
2385     public static void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
2386         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
2387         fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nexthop);
2388     }
2389
2390     public boolean isBgpConnected() {
2391         return bgpRouter.isBgpConnected();
2392     }
2393
2394     public long getLastConnectedTS() {
2395         return bgpRouter.getLastConnectedTS();
2396     }
2397
2398     public long getConnectTS() {
2399         return bgpRouter.getConnectTS();
2400     }
2401
2402     public long getStartTS() {
2403         return bgpRouter.getStartTS();
2404     }
2405
2406     public static int getTotalStaledCount() {
2407         return totalStaledCount;
2408     }
2409
2410     public static int getTotalCleared() {
2411         return totalCleared;
2412     }
2413
2414     public Timer getBgpCountersTimer() {
2415         return bgpCountersTimer;
2416     }
2417
2418     public BgpCounters getBgpCounters() {
2419         return bgpCounters;
2420     }
2421
2422     public void setBgpCountersTimer(Timer timer) {
2423         bgpCountersTimer = timer;
2424     }
2425
2426     public void setBgpAlarmsTimer(Timer timer) {
2427         bgpAlarmsTimer = timer;
2428     }
2429
2430     public void startBgpCountersTask() {
2431         if (getBgpCounters() == null) {
2432             bgpCounters = new BgpCounters(bgpConfigurationManager.getBgpSdncMipIp());
2433             setBgpCountersTimer(new Timer(true));
2434             getBgpCountersTimer().scheduleAtFixedRate(bgpCounters, 0, 120 * 1000);
2435             LOG.info("Bgp Counters task scheduled for every two minutes.");
2436
2437             bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
2438         }
2439     }
2440
2441     public void stopBgpCountersTask() {
2442         Timer timer = getBgpCountersTimer();
2443         if (getBgpCounters() != null) {
2444             timer.cancel();
2445             setBgpCountersTimer(null);
2446             bgpCounters = null;
2447         }
2448     }
2449
2450     public void startBgpAlarmsTask() {
2451         if (getBgpAlarms() == null) {
2452             bgpAlarms = new BgpAlarms(this);
2453             setBgpAlarmsTimer(new Timer(true));
2454             getBgpAlarmsTimer().scheduleAtFixedRate(bgpAlarms, 0, 60 * 1000);
2455             LOG.info("Bgp Alarms task scheduled for every minute.");
2456         }
2457     }
2458
2459     public void stopBgpAlarmsTask() {
2460         Timer timer = getBgpAlarmsTimer();
2461         if (getBgpAlarms() != null) {
2462             timer.cancel();
2463             setBgpAlarmsTimer(null);
2464             bgpAlarms = null;
2465         }
2466     }
2467
2468     public Timer getBgpAlarmsTimer() {
2469         return bgpAlarmsTimer;
2470     }
2471
2472     public BgpAlarms getBgpAlarms() {
2473         return bgpAlarms;
2474     }
2475
2476     private static String appendNextHopToPrefix(String prefix, String nextHop) {
2477         return prefix + ":" + nextHop;
2478     }
2479
2480     private static String extractPrefix(String prefixNextHop) {
2481         return prefixNextHop.split(":")[0];
2482     }
2483
2484     private static String extractNextHop(String prefixNextHop) {
2485         return prefixNextHop.split(":")[1];
2486     }
2487
2488     private static String extractMd5Secret(final Neighbors val) {
2489         String md5Secret = null;
2490         TcpSecurityOption tcpSecOpt = val.getTcpSecurityOption();
2491         if (tcpSecOpt != null) {
2492             if (tcpSecOpt instanceof TcpMd5SignatureOption) {
2493                 md5Secret = ((TcpMd5SignatureOption) tcpSecOpt).getTcpMd5SignaturePassword().getValue();
2494             } else { // unknown TcpSecurityOption
2495                 LOG.debug("neighbors  Ignored unknown tcp-security-option of peer {}", val.getAddress().getValue());
2496             }
2497         }
2498         return md5Secret;
2499     } // private method extractMd5Secret
2500 }