vpnservice/bgpmanager/bgpmanager-impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright (c) 2015 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import com.google.common.base.Optional;
  11 import io.netty.util.concurrent.GlobalEventExecutor;
  12 import java.io.BufferedReader;
  13 import java.io.File;
  14 import java.io.FileInputStream;
  15 import java.io.IOException;
  16 import java.io.InputStreamReader;
  17 import java.lang.reflect.Constructor;
  18 import java.net.InetAddress;
  19 import java.net.NetworkInterface;
  20 import java.util.Arrays;
  21 import java.util.Enumeration;
  22 import java.util.HashMap;
  23 import java.util.Iterator;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Timer;
  27 import java.util.TimerTask;
  28 import java.util.concurrent.Callable;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.CountDownLatch;
  31 import java.util.concurrent.ExecutionException;
  32 import java.util.concurrent.Executors;
  33 import java.util.concurrent.Future;
  34 import java.util.concurrent.ScheduledExecutorService;
  35 import java.util.concurrent.TimeoutException;
  36 import java.util.concurrent.atomic.AtomicBoolean;
  37 import org.opendaylight.controller.config.api.osgi.WaitingServiceTracker;
  38 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  41 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipChange;
  42 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipListener;
  43 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipService;
  44 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  45 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  46 import org.opendaylight.genius.utils.clustering.EntityOwnerUtils;
  47 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  48 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  49 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  50 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  51 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  52 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  53 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  54 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  55 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  56 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  57 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  58 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  59 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  60 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  61 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  62 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  63 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  64 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
  65 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
  67 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
  69 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
  71 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Neighbors;
  73 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsKey;
  75 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Networks;
  76 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksKey;
  78 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Vrfs;
  79 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsBuilder;
  80 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsKey;
  81 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamilies;
  82 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesKey;
  84 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihop;
  85 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihopBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSource;
  87 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSourceBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  92 import org.opendaylight.yangtools.concepts.ListenerRegistration;
  93 import org.opendaylight.yangtools.yang.binding.DataObject;
  94 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  95 import org.osgi.framework.BundleContext;
  96 import org.slf4j.Logger;
  97 import org.slf4j.LoggerFactory;
  98
  99 public class BgpConfigurationManager {
 100     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 101     private static DataBroker dataBroker;
 102     private static FibDSWriter fibDSWriter;
 103     public static IBgpManager bgpManager;
 104     private final BundleContext bundleContext;
 105     private static Bgp config;
 106     private static BgpRouter bgpRouter;
 107     private static BgpThriftService updateServer;
 108     private BgpCounters bgpCounters;
 109     private BgpAlarms bgpAlarms;
 110     private Timer bgpCountersTimer;
 111     private Timer bgpAlarmsTimer;
 112     private static final String DEF_LOGFILE = "/var/log/bgp_debug.log";
 113     private static final String DEF_LOGLEVEL = "errors";
 114     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 115     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 116     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 117     private static final String DEF_UPORT = "6644";
 118     private static final String DEF_CHOST = "127.0.0.1";
 119     private static final String DEF_CPORT = "7644";
 120     private static final String SDNC_BGP_MIP = "sdnc_bgp_mip";
 121     private static final String BGP_SDNC_MIP = "bgp_sdnc_mip";
 122     private static final String CLUSTER_CONF_FILE = "/cluster/etc/cluster.conf";
 123     private static final Timer ipActivationCheckTimer = new Timer();
 124     private static final int STALE_FIB_WAIT = 60;
 125     private static final int RESTART_DEFAULT_GR = 90;
 126     private long StaleStartTime = 0;
 127     private long StaleEndTime = 0;
 128     private long CfgReplayStartTime = 0;
 129     private long CfgReplayEndTime = 0;
 130     private long StaleCleanupTime = 0;
 131
 132     public String getBgpSdncMipIp() { return readThriftIpForCommunication(BGP_SDNC_MIP);}
 133     public long getStaleCleanupTime() {
 134         return StaleCleanupTime;
 135     }
 136
 137     public void setStaleCleanupTime(long staleCleanupTime) {
 138         StaleCleanupTime = staleCleanupTime;
 139     }
 140
 141     public long getCfgReplayEndTime() {
 142         return CfgReplayEndTime;
 143     }
 144
 145     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 146         CfgReplayEndTime = cfgReplayEndTime;
 147     }
 148
 149     public long getCfgReplayStartTime() {
 150         return CfgReplayStartTime;
 151     }
 152
 153     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 154         CfgReplayStartTime = cfgReplayStartTime;
 155     }
 156
 157     public long getStaleEndTime() {
 158         return StaleEndTime;
 159     }
 160
 161     public void setStaleEndTime(long staleEndTime) {
 162         StaleEndTime = staleEndTime;
 163     }
 164
 165     public long getStaleStartTime() {
 166         return StaleStartTime;
 167     }
 168
 169     public void setStaleStartTime(long staleStartTime) {
 170         StaleStartTime = staleStartTime;
 171     }
 172
 173
 174     // to have stale FIB map (RD, Prefix)
 175     //  number of seconds wait for route sync-up between ODL and BGP.
 176     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 360;
 177
 178     static String odlThriftIp = "127.0.0.1";
 179     static String bgpThriftIp = "127.0.0.1";
 180     private static String cHostStartup;
 181     private static String cPortStartup;
 182     private static CountDownLatch initer = new CountDownLatch(1);
 183     //static IITMProvider itmProvider;
 184     //map<rd, map<prefix/len, nexthop/label>>
 185     private static Map<String, Map<String, String>> staledFibEntriesMap = new ConcurrentHashMap<>();
 186
 187     static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 188     static final String BGP_ENTITY_NAME = "bgp";
 189
 190     static int totalStaledCount = 0;
 191     static int totalCleared = 0;
 192
 193     private static final Class[] reactors = {
 194             ConfigServerReactor.class, AsIdReactor.class,
 195             GracefulRestartReactor.class, LoggingReactor.class,
 196             NeighborsReactor.class, UpdateSourceReactor.class,
 197             EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 198             NetworksReactor.class, VrfsReactor.class, BgpReactor.class
 199     };
 200
 201     private ListenerRegistration<DataChangeListener>[] registrations;
 202
 203     final BgpConfigurationManager bgpConfigurationManager;
 204
 205     public BgpConfigurationManager(final DataBroker dataBroker,
 206                                    final EntityOwnershipService entityOwnershipService,
 207                                    final FibDSWriter fibDSWriter,
 208                                    final BundleContext bundleContext)
 209             throws InterruptedException, ExecutionException, TimeoutException {
 210         BgpConfigurationManager.dataBroker = dataBroker;
 211         BgpConfigurationManager.fibDSWriter = fibDSWriter;
 212         this.bundleContext = bundleContext;
 213         String uPort = getProperty(UPDATE_PORT, DEF_UPORT);
 214         cHostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 215         cPortStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 216         LOG.info("UpdateServer at localhost:" + uPort + " ConfigServer at "
 217                 + cHostStartup + ":" + cPortStartup);
 218         VtyshCli.setHostAddr(cHostStartup);
 219         ClearBgpCli.setHostAddr(cHostStartup);
 220         setEntityOwnershipService(entityOwnershipService);
 221         bgpRouter = BgpRouter.getInstance();
 222         odlThriftIp = readThriftIpForCommunication(SDNC_BGP_MIP);
 223         bgpThriftIp = readThriftIpForCommunication(BGP_SDNC_MIP);
 224         registerCallbacks();
 225
 226         LOG.info("BGP Configuration manager initialized");
 227         initer.countDown();
 228
 229         bgpConfigurationManager = this;
 230         BgpUtil.batchSize = BgpUtil.BATCH_SIZE;
 231         if (Integer.getInteger("batch.size") != null) {
 232             BgpUtil.batchSize = Integer.getInteger("batch.size");
 233         }
 234         BgpUtil.batchInterval = BgpUtil.PERIODICITY;
 235         if (Integer.getInteger("batch.wait.time") != null) {
 236             BgpUtil.batchInterval = Integer.getInteger("batch.wait.time");
 237         }
 238         BgpUtil.registerWithBatchManager(new BgpVrfBatchHandler());
 239
 240         GlobalEventExecutor.INSTANCE.execute(new Runnable() {
 241             @Override
 242             public void run() {
 243                 final WaitingServiceTracker<IBgpManager> tracker = WaitingServiceTracker.create(
 244                         IBgpManager.class, bundleContext);
 245                 bgpManager = tracker.waitForService(WaitingServiceTracker.FIVE_MINUTES);
 246                 updateServer = new BgpThriftService(Integer.parseInt(uPort), bgpManager, fibDSWriter);
 247                 updateServer.start();
 248                 LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 249             }
 250         });
 251     }
 252
 253     private Object createListener(Class<?> cls) {
 254         Constructor<?> ctor;
 255         Object obj = null;
 256
 257         try {
 258             ctor = cls.getConstructor(BgpConfigurationManager.class);
 259             obj = ctor.newInstance(this);
 260         } catch (Exception e) {
 261             LOG.error("Failed to create listener object", e);
 262         }
 263         return obj;
 264     }
 265
 266     private void registerCallbacks() {
 267         String emsg = "Failed to register listener";
 268         registrations = new ListenerRegistration[reactors.length];
 269         InstanceIdentifier<?> iid = InstanceIdentifier.create(Bgp.class);
 270         for (Class reactor : reactors) {
 271             Object obj = createListener(reactor);
 272             String dclName = obj.getClass().getName();
 273             try {
 274                 AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 275                 dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 276             } catch (Exception e) {
 277                 LOG.error(emsg, e);
 278                 throw new IllegalStateException(emsg + " " + dclName, e);
 279             }
 280         }
 281     }
 282
 283     public void close() {
 284         if (updateServer != null) {
 285             updateServer.stop();
 286         }
 287         LOG.info("{} close", getClass().getSimpleName());
 288     }
 289
 290     private boolean configExists() throws InterruptedException, ExecutionException, TimeoutException {
 291         InstanceIdentifier.InstanceIdentifierBuilder<Bgp> iib =
 292                 InstanceIdentifier.builder(Bgp.class);
 293         InstanceIdentifier<Bgp> iid = iib.build();
 294         Optional<Bgp> b = BgpUtil.read(dataBroker,
 295                 LogicalDatastoreType.CONFIGURATION, iid);
 296         return b.isPresent();
 297     }
 298
 299     private String getProperty(String var, String def) {
 300         String s = bundleContext.getProperty(var);
 301         return (s == null ? def : s);
 302     }
 303
 304     boolean ignoreClusterDcnEventForFollower() {
 305         return !EntityOwnerUtils.amIEntityOwner(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME);
 306     }
 307
 308     public Bgp get() {
 309         config = getConfig();
 310         return config;
 311     }
 312
 313     public void setEntityOwnershipService(final EntityOwnershipService entityOwnershipService) {
 314         try {
 315             EntityOwnerUtils.registerEntityCandidateForOwnerShip(entityOwnershipService,
 316                     BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME, new EntityOwnershipListener() {
 317                 @Override
 318                 public void ownershipChanged(EntityOwnershipChange ownershipChange) {
 319                     LOG.trace("entity owner change event fired");
 320                     if (ownershipChange.hasOwner() && ownershipChange.isOwner()) {
 321                         LOG.trace("This PL is the Owner");
 322                         activateMIP();
 323                         bgpRestarted();
 324                     } else {
 325                         LOG.info("Not owner: hasOwner: {}, isOwner: {}",ownershipChange.hasOwner(),
 326                                 ownershipChange.isOwner() );
 327                     }
 328                 }
 329             });
 330         } catch (Exception e) {
 331             LOG.error("failed to register bgp entity", e);
 332         }
 333     }
 334
 335     private static final String addWarn =
 336             "Config store updated; undo with Delete if needed.";
 337     private static final String delWarn =
 338             "Config store updated; undo with Add if needed.";
 339     private static final String updWarn =
 340             "Update operation not supported; Config store updated;"
 341                     + " restore with another Update if needed.";
 342
 343     public class ConfigServerReactor
 344             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 345             implements AutoCloseable, ClusteredDataTreeChangeListener <ConfigServer> {
 346         private static final String yangObj = "config-server ";
 347
 348         public ConfigServerReactor() {
 349             super(ConfigServer.class, ConfigServerReactor.class);
 350         }
 351
 352         protected synchronized void
 353         add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 354             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 355             if (ignoreClusterDcnEventForFollower()) {
 356                 return;
 357             }
 358
 359             try {
 360                 initer.await();
 361             } catch (Exception e) {
 362             }
 363             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 364             synchronized (BgpConfigurationManager.this) {
 365                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 366                         val.getPort().intValue());
 367                 if (!res) {
 368                     LOG.error(yangObj + "Add failed; " + addWarn);
 369                 }
 370             }
 371         }
 372
 373         @Override
 374         protected ConfigServerReactor getDataTreeChangeListener() {
 375             return ConfigServerReactor.this;
 376         }
 377
 378         @Override
 379         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 380             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 381         }
 382
 383         protected synchronized void
 384         remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 385             LOG.trace("received bgp disconnect");
 386             if (ignoreClusterDcnEventForFollower()) {
 387                 return;
 388             }
 389             synchronized (BgpConfigurationManager.this) {
 390                 bgpRouter.disconnect();
 391             }
 392         }
 393
 394         protected void update(InstanceIdentifier<ConfigServer> iid,
 395                               ConfigServer oldval, ConfigServer newval) {
 396             LOG.trace("received bgp Connection update");
 397             if (ignoreClusterDcnEventForFollower()) {
 398                 return;
 399             }
 400             LOG.error(yangObj + updWarn);
 401         }
 402
 403         @Override
 404         public void close() {
 405             try {
 406                 super.close();
 407             } catch (Exception e) {
 408                 LOG.error("ConfigServerReactor failed to close: ", e);
 409             }
 410         }
 411     }
 412
 413     private BgpRouter getClient(String yangObj) {
 414         if (bgpRouter == null) {
 415             LOG.warn(yangObj + ": configuration received when BGP is inactive");
 416         }
 417         return bgpRouter;
 418     }
 419
 420     public class AsIdReactor
 421             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 422             implements AutoCloseable, ClusteredDataTreeChangeListener<AsId> {
 423
 424         private static final String yangObj = "as-id ";
 425
 426         public AsIdReactor() {
 427             super(AsId.class, AsIdReactor.class);
 428         }
 429
 430         protected synchronized void
 431         add(InstanceIdentifier<AsId> iid, AsId val) {
 432             LOG.error("received bgp add asid {}",val);
 433             if (ignoreClusterDcnEventForFollower()) {
 434                 return;
 435             }
 436             LOG.debug("received add router config asNum {}", val.getLocalAs().intValue());
 437             synchronized (BgpConfigurationManager.this) {
 438                 BgpRouter br = getClient(yangObj);
 439                 if (br == null) {
 440                     LOG.error("no bgp router client found exiting asid add");
 441                     return;
 442                 }
 443                 int asNum = val.getLocalAs().intValue();
 444                 Ipv4Address routerId = val.getRouterId();
 445                 Boolean afb = val.isAnnounceFbit();
 446                 String rid = (routerId == null) ? "" : routerId.getValue();
 447                 int stalepathTime = (int) getStalePathtime(RESTART_DEFAULT_GR, val);
 448                 boolean announceFbit = (afb == null) ? false : afb.booleanValue();
 449                 try {
 450                     br.startBgp(asNum, rid, stalepathTime, announceFbit);
 451                     if (getBgpCounters() == null) {
 452                         startBgpCountersTask();
 453                     }
 454                     if (getBgpAlarms() == null) {
 455                         startBgpAlarmsTask();
 456                     }
 457                 } catch (BgpRouterException bre) {
 458                     if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
 459                         LOG.error(yangObj + "Add requested when BGP is already active");
 460                     } else {
 461                         LOG.error(yangObj + "Add received exception: \""
 462                                 + bre + "\"; " + addWarn);
 463                     }
 464                 } catch (Exception e) {
 465                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; " + addWarn);
 466                 }
 467             }
 468         }
 469
 470         @Override
 471         protected AsIdReactor getDataTreeChangeListener() {
 472             return AsIdReactor.this;
 473         }
 474
 475         @Override
 476         protected InstanceIdentifier<AsId> getWildCardPath() {
 477             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 478         }
 479
 480         protected synchronized void
 481         remove(InstanceIdentifier<AsId> iid, AsId val) {
 482             LOG.error("received delete router config asNum {}", val.getLocalAs().intValue());
 483             if (ignoreClusterDcnEventForFollower()) {
 484                 return;
 485             }
 486             synchronized (BgpConfigurationManager.this) {
 487                 BgpRouter br = getClient(yangObj);
 488                 if (br == null) {
 489                     return;
 490                 }
 491                 int asNum = val.getLocalAs().intValue();
 492                 try {
 493                     br.stopBgp(asNum);
 494                 } catch (Exception e) {
 495                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; " + delWarn);
 496                 }
 497                 if (getBgpCounters() != null) {
 498                     stopBgpCountersTask();
 499                 }
 500                 if (getBgpAlarms() != null) {
 501                     stopBgpAlarmsTask();
 502                 }
 503             }
 504         }
 505
 506         protected void update(InstanceIdentifier<AsId> iid,
 507                               AsId oldval, AsId newval) {
 508             if (ignoreClusterDcnEventForFollower()) {
 509                 return;
 510             }
 511             LOG.error(yangObj + updWarn);
 512         }
 513
 514         @Override
 515         public void close() {
 516             try {
 517                 super.close();
 518             } catch (Exception e) {
 519                 e.printStackTrace();
 520             }
 521         }
 522     }
 523
 524     public class GracefulRestartReactor
 525             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 526             implements AutoCloseable, ClusteredDataTreeChangeListener<GracefulRestart> {
 527
 528         private static final String yangObj = "graceful-restart ";
 529
 530         public GracefulRestartReactor() {
 531             super(GracefulRestart.class, GracefulRestartReactor.class);
 532         }
 533
 534         protected synchronized void
 535         add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 536             if (ignoreClusterDcnEventForFollower()) {
 537                 return;
 538             }
 539             synchronized (BgpConfigurationManager.this) {
 540                 BgpRouter br = getClient(yangObj);
 541                 if (br == null) {
 542                     return;
 543                 }
 544                 try {
 545                     br.addGracefulRestart(val.getStalepathTime().intValue());
 546                 } catch (Exception e) {
 547                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; " + addWarn);
 548                 }
 549             }
 550         }
 551
 552         @Override
 553         protected GracefulRestartReactor getDataTreeChangeListener() {
 554             return GracefulRestartReactor.this;
 555         }
 556
 557         @Override
 558         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 559             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 560         }
 561
 562         protected synchronized void
 563         remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 564             if (ignoreClusterDcnEventForFollower()) {
 565                 return;
 566             }
 567             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 568             synchronized (BgpConfigurationManager.this) {
 569                 BgpRouter br = getClient(yangObj);
 570                 if (br == null) {
 571                     return;
 572                 }
 573                 try {
 574                     br.delGracefulRestart();
 575                 } catch (Exception e) {
 576                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 577                             + delWarn);
 578                 }
 579             }
 580         }
 581
 582         protected void update(InstanceIdentifier<GracefulRestart> iid,
 583                               GracefulRestart oldval, GracefulRestart newval) {
 584             if (ignoreClusterDcnEventForFollower()) {
 585                 return;
 586             }
 587             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 588             synchronized (BgpConfigurationManager.this) {
 589                 BgpRouter br = getClient(yangObj);
 590                 if (br == null) {
 591                     return;
 592                 }
 593                 try {
 594                     br.addGracefulRestart(newval.getStalepathTime().intValue());
 595                 } catch (Exception e) {
 596                     LOG.error(yangObj + "update received exception: \"" + e + "\"; " + addWarn);
 597                 }
 598             }
 599         }
 600
 601         @Override
 602         public void close() {
 603             try {
 604                 super.close();
 605             } catch (Exception e) {
 606                 e.printStackTrace();
 607             }
 608         }
 609     }
 610
 611     public class LoggingReactor
 612             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 613             implements AutoCloseable, ClusteredDataTreeChangeListener<Logging> {
 614
 615         private static final String yangObj = "logging ";
 616
 617         public LoggingReactor() {
 618             super(Logging.class, LoggingReactor.class);
 619         }
 620
 621         protected synchronized void
 622         add(InstanceIdentifier<Logging> iid, Logging val) {
 623             if (ignoreClusterDcnEventForFollower()) {
 624                 return;
 625             }
 626             synchronized (BgpConfigurationManager.this) {
 627                 BgpRouter br = getClient(yangObj);
 628                 if (br == null) {
 629                     return;
 630                 }
 631                 try {
 632                     br.setLogging(val.getFile(), val.getLevel());
 633                 } catch (Exception e) {
 634                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 635                             + addWarn);
 636                 }
 637             }
 638         }
 639
 640         @Override
 641         protected LoggingReactor getDataTreeChangeListener() {
 642             return LoggingReactor.this;
 643         }
 644
 645         @Override
 646         protected InstanceIdentifier<Logging> getWildCardPath() {
 647             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 648         }
 649
 650         protected synchronized void
 651         remove(InstanceIdentifier<Logging> iid, Logging val) {
 652             if (ignoreClusterDcnEventForFollower()) {
 653                 return;
 654             }
 655             LOG.debug("received remove Logging config val {}", val.getLevel());
 656             synchronized (BgpConfigurationManager.this) {
 657                 BgpRouter br = getClient(yangObj);
 658                 if (br == null) {
 659                     return;
 660                 }
 661                 try {
 662                     br.setLogging(DEF_LOGFILE, DEF_LOGLEVEL);
 663                 } catch (Exception e) {
 664                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 665                             + delWarn);
 666                 }
 667             }
 668         }
 669
 670         protected void update(InstanceIdentifier<Logging> iid,
 671                               Logging oldval, Logging newval) {
 672             if (ignoreClusterDcnEventForFollower()) {
 673                 return;
 674             }
 675             synchronized (BgpConfigurationManager.this) {
 676                 BgpRouter br = getClient(yangObj);
 677                 if (br == null) {
 678                     return;
 679                 }
 680                 try {
 681                     br.setLogging(newval.getFile(), newval.getLevel());
 682                 } catch (Exception e) {
 683                     LOG.error(yangObj + "newval received exception: \"" + e + "\"; "
 684                             + addWarn);
 685                 }
 686             }
 687         }
 688
 689         @Override
 690         public void close() {
 691             try {
 692                 super.close();
 693             } catch (Exception e) {
 694                 e.printStackTrace();
 695             }
 696         }
 697     }
 698
 699     public class NeighborsReactor
 700             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 701             implements AutoCloseable, ClusteredDataTreeChangeListener<Neighbors> {
 702
 703         private static final String yangObj = "neighbors ";
 704
 705         public NeighborsReactor() {
 706             super(Neighbors.class, NeighborsReactor.class);
 707         }
 708
 709         protected synchronized void
 710         add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 711             if (ignoreClusterDcnEventForFollower()) {
 712                 return;
 713             }
 714             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 715             synchronized (BgpConfigurationManager.this) {
 716                 BgpRouter br = getClient(yangObj);
 717                 if (br == null) {
 718                     return;
 719                 }
 720                 String peerIp = val.getAddress().getValue();
 721                 int as = val.getRemoteAs().intValue();
 722                 try {
 723                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 724                     br.addNeighbor(peerIp, as);
 725
 726                 } catch (Exception e) {
 727                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 728                             + addWarn);
 729                 }
 730             }
 731         }
 732
 733         @Override
 734         protected NeighborsReactor getDataTreeChangeListener() {
 735             return NeighborsReactor.this;
 736         }
 737
 738         @Override
 739         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 740             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class);
 741         }
 742
 743         protected synchronized void
 744         remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 745             if (ignoreClusterDcnEventForFollower()) {
 746                 return;
 747             }
 748             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 749             synchronized (BgpConfigurationManager.this) {
 750                 BgpRouter br = getClient(yangObj);
 751                 if (br == null) {
 752                     return;
 753                 }
 754                 String peerIp = val.getAddress().getValue();
 755                 try {
 756                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 757                     br.delNeighbor(peerIp);
 758                 } catch (Exception e) {
 759                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 760                             + delWarn);
 761                 }
 762             }
 763         }
 764
 765         protected void update(InstanceIdentifier<Neighbors> iid,
 766                               Neighbors oldval, Neighbors newval) {
 767             if (ignoreClusterDcnEventForFollower()) {
 768                 return;
 769             }
 770             //purposefully nothing to do.
 771         }
 772
 773         @Override
 774         public void close() {
 775             try {
 776                 super.close();
 777             } catch (Exception e) {
 778                 e.printStackTrace();
 779             }
 780         }
 781     }
 782
 783     public class EbgpMultihopReactor
 784             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 785             implements AutoCloseable, ClusteredDataTreeChangeListener<EbgpMultihop> {
 786
 787         private static final String yangObj = "ebgp-multihop ";
 788
 789         public EbgpMultihopReactor() {
 790             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 791         }
 792
 793         protected synchronized void
 794         add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 795             if (ignoreClusterDcnEventForFollower()) {
 796                 return;
 797             }
 798             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 799             synchronized (BgpConfigurationManager.this) {
 800                 BgpRouter br = getClient(yangObj);
 801                 if (br == null) {
 802                     return;
 803                 }
 804                 String peerIp = val.getPeerIp().getValue();
 805                 try {
 806                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 807                 } catch (Exception e) {
 808                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 809                             + addWarn);
 810                 }
 811             }
 812         }
 813
 814         @Override
 815         protected EbgpMultihopReactor getDataTreeChangeListener() {
 816             return EbgpMultihopReactor.this;
 817         }
 818
 819         @Override
 820         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
 821             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(EbgpMultihop.class);
 822         }
 823
 824         protected synchronized void
 825         remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 826             if (ignoreClusterDcnEventForFollower()) {
 827                 return;
 828             }
 829             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 830             synchronized (BgpConfigurationManager.this) {
 831                 BgpRouter br = getClient(yangObj);
 832                 if (br == null) {
 833                     return;
 834                 }
 835                 String peerIp = val.getPeerIp().getValue();
 836                 try {
 837                     br.delEbgpMultihop(peerIp);
 838                 } catch (Exception e) {
 839                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 840                             + delWarn);
 841                 }
 842             }
 843         }
 844
 845         protected void update(InstanceIdentifier<EbgpMultihop> iid,
 846                               EbgpMultihop oldval, EbgpMultihop newval) {
 847             if (ignoreClusterDcnEventForFollower()) {
 848                 return;
 849             }
 850             LOG.error(yangObj + updWarn);
 851         }
 852
 853         @Override
 854         public void close() {
 855             try {
 856                 super.close();
 857             } catch (Exception e) {
 858                 e.printStackTrace();
 859             }
 860         }
 861     }
 862
 863     public class UpdateSourceReactor
 864             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
 865             implements AutoCloseable, ClusteredDataTreeChangeListener<UpdateSource> {
 866
 867         private static final String yangObj = "update-source ";
 868
 869         public UpdateSourceReactor() {
 870             super(UpdateSource.class, UpdateSourceReactor.class);
 871         }
 872
 873         protected synchronized void
 874         add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 875             if (ignoreClusterDcnEventForFollower()) {
 876                 return;
 877             }
 878             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
 879             synchronized (BgpConfigurationManager.this) {
 880                 BgpRouter br = getClient(yangObj);
 881                 if (br == null) {
 882                     return;
 883                 }
 884                 String peerIp = val.getPeerIp().getValue();
 885                 try {
 886                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
 887                 } catch (Exception e) {
 888                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 889                             + addWarn);
 890                 }
 891             }
 892         }
 893
 894         @Override
 895         protected UpdateSourceReactor getDataTreeChangeListener() {
 896             return UpdateSourceReactor.this;
 897         }
 898
 899         @Override
 900         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
 901             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(UpdateSource.class);
 902         }
 903
 904         protected synchronized void
 905         remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 906             if (ignoreClusterDcnEventForFollower()) {
 907                 return;
 908             }
 909             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
 910             synchronized (BgpConfigurationManager.this) {
 911                 BgpRouter br = getClient(yangObj);
 912                 if (br == null) {
 913                     return;
 914                 }
 915                 String peerIp = val.getPeerIp().getValue();
 916                 try {
 917                     br.delUpdateSource(peerIp);
 918                 } catch (Exception e) {
 919                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
 920                             + delWarn);
 921                 }
 922             }
 923         }
 924
 925         protected void update(InstanceIdentifier<UpdateSource> iid,
 926                               UpdateSource oldval, UpdateSource newval) {
 927             if (ignoreClusterDcnEventForFollower()) {
 928                 return;
 929             }
 930             LOG.error(yangObj + updWarn);
 931         }
 932
 933         @Override
 934         public void close() {
 935             try {
 936                 super.close();
 937             } catch (Exception e) {
 938                 e.printStackTrace();
 939             }
 940         }
 941     }
 942
 943     public class AddressFamiliesReactor
 944             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
 945             implements AutoCloseable, ClusteredDataTreeChangeListener<AddressFamilies> {
 946
 947         private static final String yangObj = "address-families ";
 948
 949         public AddressFamiliesReactor() {
 950             super(AddressFamilies.class, AddressFamiliesReactor.class);
 951         }
 952
 953         protected synchronized void
 954         add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
 955             if (ignoreClusterDcnEventForFollower()) {
 956                 return;
 957             }
 958             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
 959             synchronized (BgpConfigurationManager.this) {
 960                 BgpRouter br = getClient(yangObj);
 961                 if (br == null) {
 962                     return;
 963                 }
 964                 String peerIp = val.getPeerIp().getValue();
 965                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
 966                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
 967                 try {
 968                     br.addAddressFamily(peerIp, afi, safi);
 969                 } catch (Exception e) {
 970                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
 971                             + addWarn);
 972                 }
 973             }
 974         }
 975
 976         @Override
 977         protected AddressFamiliesReactor getDataTreeChangeListener() {
 978             return AddressFamiliesReactor.this;
 979         }
 980
 981         @Override
 982         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
 983             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(AddressFamilies.class);
 984         }
 985
 986         protected synchronized void
 987         remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
 988             if (ignoreClusterDcnEventForFollower()) {
 989                 return;
 990             }
 991             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
 992             synchronized (BgpConfigurationManager.this) {
 993                 BgpRouter br = getClient(yangObj);
 994                 if (br == null) {
 995                     return;
 996                 }
 997                 String peerIp = val.getPeerIp().getValue();
 998                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
 999                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1000                 try {
1001                     br.delAddressFamily(peerIp, afi, safi);
1002                 } catch (Exception e) {
1003                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
1004                             + delWarn);
1005                 }
1006             }
1007         }
1008
1009         protected void update(InstanceIdentifier<AddressFamilies> iid,
1010                               AddressFamilies oldval, AddressFamilies newval) {
1011             if (ignoreClusterDcnEventForFollower()) {
1012                 return;
1013             }
1014             LOG.error(yangObj + updWarn);
1015         }
1016
1017         @Override
1018         public void close() {
1019             try {
1020                 super.close();
1021             } catch (Exception e) {
1022                 e.printStackTrace();
1023             }
1024         }
1025     }
1026
1027     public class NetworksReactor
1028             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
1029             implements AutoCloseable, ClusteredDataTreeChangeListener<Networks> {
1030
1031         private static final String yangObj = "networks ";
1032
1033         public NetworksReactor() {
1034             super(Networks.class, NetworksReactor.class);
1035         }
1036         @Override
1037         public NetworksReactor getDataTreeChangeListener() {
1038             return NetworksReactor.this;
1039         }
1040
1041         protected synchronized void
1042         add(InstanceIdentifier<Networks> iid, Networks val) {
1043             if (ignoreClusterDcnEventForFollower()) {
1044                 return;
1045             }
1046             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1047             synchronized (BgpConfigurationManager.this) {
1048                 BgpRouter br = getClient(yangObj);
1049                 if (br == null) {
1050                     return;
1051                 }
1052                 String rd = val.getRd();
1053                 String pfxlen = val.getPrefixLen();
1054                 String nh = val.getNexthop().getValue();
1055                 Long label = val.getLabel();
1056                 int lbl = (label == null) ? qbgpConstants.LBL_NO_LABEL
1057                         : label.intValue();
1058                 try {
1059                     br.addPrefix(rd, pfxlen, nh, lbl);
1060                 } catch (Exception e) {
1061                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; " + addWarn);
1062                 }
1063             }
1064         }
1065
1066         @Override
1067         protected InstanceIdentifier<Networks> getWildCardPath() {
1068             return InstanceIdentifier.create(Bgp.class).child(Networks.class);
1069         }
1070
1071         protected synchronized void
1072         remove(InstanceIdentifier<Networks> iid, Networks val) {
1073             if (ignoreClusterDcnEventForFollower()) {
1074                 return;
1075             }
1076             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1077             synchronized (BgpConfigurationManager.this) {
1078                 BgpRouter br = getClient(yangObj);
1079                 if (br == null) {
1080                     return;
1081                 }
1082                 String rd = val.getRd();
1083                 String pfxlen = val.getPrefixLen();
1084                 Long label = val.getLabel();
1085                 int lbl = (label == null) ? 0 : label.intValue();
1086                 if (rd == null && lbl > 0) {
1087                     //LU prefix is being deleted.
1088                     rd = Integer.toString(lbl);
1089                 }
1090                 try {
1091                     br.delPrefix(rd, pfxlen);
1092                 } catch (Exception e) {
1093                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
1094                             + delWarn);
1095                 }
1096             }
1097         }
1098
1099         protected void update(final InstanceIdentifier<Networks> iid,
1100                               final Networks oldval, final Networks newval) {
1101             if (ignoreClusterDcnEventForFollower()) {
1102                 return;
1103             }
1104             LOG.debug("received update networks config val {}", newval.getPrefixLen());
1105             remove(iid, oldval);
1106             timer.schedule(new TimerTask() {
1107                 @Override
1108                 public void run() {
1109                     add(iid, newval);
1110                 }
1111             }, Integer.getInteger("bgp.nexthop.update.delay.in.secs", 5) * 1000);
1112         }
1113
1114         @Override
1115         public void close() {
1116             try {
1117                 super.close();
1118             } catch (Exception e) {
1119                 e.printStackTrace();
1120             }
1121         }
1122     }
1123
1124     static Timer timer = new Timer();
1125
1126     public class VrfsReactor
1127             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1128             implements AutoCloseable, ClusteredDataTreeChangeListener<Vrfs> {
1129
1130         private static final String yangObj = "vrfs ";
1131
1132         public VrfsReactor() {
1133             super(Vrfs.class, VrfsReactor.class);
1134         }
1135
1136         protected synchronized void
1137         add(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1138             if (ignoreClusterDcnEventForFollower()) {
1139                 return;
1140             }
1141             LOG.debug("received add Vrfs config val {}", val.getRd());
1142             synchronized (BgpConfigurationManager.this) {
1143                 BgpRouter br = getClient(yangObj);
1144                 if (br == null) {
1145                     return;
1146                 }
1147                 try {
1148                     br.addVrf(val.getRd(), val.getImportRts(),
1149                             val.getExportRts());
1150                 } catch (Exception e) {
1151                     LOG.error(yangObj + "Add received exception: \"" + e + "\"; "
1152                             + addWarn);
1153                 }
1154             }
1155         }
1156
1157         @Override
1158         protected VrfsReactor getDataTreeChangeListener() {
1159             return VrfsReactor.this;
1160         }
1161
1162         @Override
1163         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1164             return InstanceIdentifier.create(Bgp.class).child(Vrfs.class);
1165         }
1166
1167         protected synchronized void
1168         remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1169             if (ignoreClusterDcnEventForFollower()) {
1170                 return;
1171             }
1172             LOG.debug("received remove Vrfs config val {}", val.getRd());
1173             synchronized (BgpConfigurationManager.this) {
1174                 BgpRouter br = getClient(yangObj);
1175                 if (br == null) {
1176                     return;
1177                 }
1178                 try {
1179                     br.delVrf(val.getRd());
1180                 } catch (Exception e) {
1181                     LOG.error(yangObj + " Delete received exception:  \"" + e + "\"; "
1182                             + delWarn);
1183                 }
1184             }
1185         }
1186
1187         protected void update(InstanceIdentifier<Vrfs> iid,
1188                               Vrfs oldval, Vrfs newval) {
1189             if (ignoreClusterDcnEventForFollower()) {
1190                 return;
1191             }
1192             LOG.debug("VRFS: Update getting triggered for VRFS rd {}", oldval.getRd());
1193             LOG.error(yangObj + updWarn);
1194         }
1195
1196         @Override
1197         public void close() {
1198             try {
1199                 super.close();
1200             } catch (Exception e) {
1201                 e.printStackTrace();
1202             }
1203         }
1204     }
1205
1206     Future lastCleanupJob;
1207     Future lastReplayJobFt = null;
1208     protected void activateMIP() {
1209         try {
1210             LOG.trace("BgpReactor: Executing MIP Activate command");
1211             Process process_bgp = Runtime.getRuntime().exec("cluster ip -a sdnc_bgp_mip");
1212             Process process_os = Runtime.getRuntime().exec("cluster ip -a sdnc_os_mip");
1213             LOG.trace("bgpMIP Activated");
1214
1215         } catch (IOException io) {
1216             //LOG.error("IO Exception got while activating mip:  ", io);
1217         } catch (Exception e) {
1218             //LOG.error("Exception got while activating mip: ", e);
1219         }
1220     }
1221
1222     AtomicBoolean started = new AtomicBoolean(false);
1223
1224     public class BgpReactor
1225             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1226             implements AutoCloseable, ClusteredDataTreeChangeListener<Bgp> {
1227
1228         private static final String yangObj = "Bgp ";
1229
1230         public BgpReactor() {
1231             super(Bgp.class, BgpReactor.class);
1232         }
1233
1234
1235         protected synchronized void
1236         add(InstanceIdentifier<Bgp> iid, Bgp val) {
1237             LOG.error("received add Bgp config replaying the config");
1238
1239             try {
1240                 initer.await();
1241             } catch (Exception e) {
1242             }
1243             synchronized (BgpConfigurationManager.this) {
1244                 config = val;
1245                 if (ignoreClusterDcnEventForFollower()) {
1246                     return;
1247                 }
1248                 activateMIP();
1249                 if (isIpAvailable(odlThriftIp)) {
1250                     bgpRestarted();
1251                 } else {
1252                     ipActivationCheckTimer.scheduleAtFixedRate(new TimerTask() {
1253                         @Override
1254                         public void run() {
1255                             if (isIpAvailable(odlThriftIp)) {
1256                                 bgpRestarted();
1257                                 ipActivationCheckTimer.cancel();
1258                             } else {
1259                                 LOG.trace("waiting for odlThriftIP: {} to be present", odlThriftIp);
1260                             }
1261                         }
1262                     }, 10000L, 10000L);
1263                 }
1264             }
1265         }
1266
1267         @Override
1268         protected BgpReactor getDataTreeChangeListener() {
1269             return BgpReactor.this;
1270         }
1271
1272         @Override
1273         protected InstanceIdentifier<Bgp> getWildCardPath() {
1274             return InstanceIdentifier.create(Bgp.class);
1275         }
1276
1277         protected synchronized void
1278         remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1279             if (ignoreClusterDcnEventForFollower()) {
1280                 return;
1281             }
1282             LOG.debug("received remove Bgp config");
1283             synchronized (BgpConfigurationManager.this) {
1284                 config = null;
1285             }
1286         }
1287
1288         protected void update(InstanceIdentifier<Bgp> iid,
1289                               Bgp oldval, Bgp newval) {
1290             if (ignoreClusterDcnEventForFollower()) {
1291                 return;
1292             }
1293             synchronized (BgpConfigurationManager.this) {
1294                 config = newval;
1295             }
1296         }
1297
1298         @Override
1299         public void close() {
1300             try {
1301                 super.close();
1302             } catch (Exception e) {
1303                 e.printStackTrace();
1304             }
1305         }
1306     }
1307
1308     public String readThriftIpForCommunication( String mipAddr) {
1309         File f = new File(CLUSTER_CONF_FILE);
1310         if (!f.exists()) {
1311             return  DEF_CHOST;
1312         }
1313         BufferedReader br = null;
1314         try {
1315             br = new BufferedReader(new InputStreamReader(
1316                     new FileInputStream(f)));
1317             String line = br.readLine();
1318             while (line != null) {
1319                 if (line.contains(mipAddr)) {
1320                     line = line.trim();
1321                     return line.substring(line.lastIndexOf(" ") + 1);
1322                 }
1323                 line = br.readLine();
1324             }
1325         } catch (Exception e) {
1326         } finally {
1327             try {
1328                 br.close();
1329             } catch (Exception ignore) {
1330             }
1331         }
1332         return DEF_CHOST;
1333     }
1334
1335     public boolean isIpAvailable(String odlip) {
1336
1337         try {
1338             if (odlip != null) {
1339                 if ("127.0.0.1".equals(odlip)) {
1340                     return true;
1341                 }
1342                 Enumeration e = NetworkInterface.getNetworkInterfaces();
1343                 while (e.hasMoreElements()) {
1344                     NetworkInterface n = (NetworkInterface) e.nextElement();
1345                     Enumeration ee = n.getInetAddresses();
1346                     while (ee.hasMoreElements()) {
1347                         InetAddress i = (InetAddress) ee.nextElement();
1348                         if (odlip.equals(i.getHostAddress())) {
1349                             return true;
1350                         }
1351                         ;
1352                     }
1353                 }
1354             }
1355         } catch (Exception e) {
1356         }
1357         return false;
1358     }
1359
1360     public static long getStalePathtime(int defValue, AsId as_num) {
1361         long spt = 0;
1362         try {
1363             spt = getConfig().getGracefulRestart().getStalepathTime();
1364         } catch (Exception e) {
1365             try {
1366                 spt = as_num.getStalepathTime();
1367                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1368             } catch (Exception ignore) {
1369                 LOG.trace("BGP AS id is not set using graceful");
1370                 spt = defValue;
1371             }
1372         }
1373         if (spt == 0) {
1374             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1375             spt = defValue;
1376         }
1377         return spt;
1378     }
1379
1380     public synchronized void bgpRestarted() {
1381         /*
1382          * If there a thread which in the process of stale cleanup, cancel it
1383          * and start a new thread (to avoid processing same again).
1384          */
1385         if (previousReplayJobInProgress()) {
1386             cancelPreviousReplayJob();
1387         }
1388         Runnable task = new Runnable() {
1389             @Override
1390             public void run() {
1391                 try {
1392                     LOG.info("running bgp replay task ");
1393                     if (get() == null) {
1394                         String host = getConfigHost();
1395                         int port = getConfigPort();
1396                         LOG.info("connecting  to bgp host {} ", host);
1397
1398                         boolean res = bgpRouter.connect(host, port);
1399                         LOG.info("no config to push in bgp replay task ");
1400                         return;
1401                     }
1402                     setStaleStartTime(System.currentTimeMillis());
1403                     LOG.info("started creating stale fibDSWriter  map ");
1404                     createStaleFibMap();
1405                     setStaleEndTime(System.currentTimeMillis());
1406                     LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime()- getStaleStartTime());
1407                     LOG.info("started bgp config replay ");
1408                     setCfgReplayStartTime(System.currentTimeMillis());
1409                     replay();
1410                     setCfgReplayEndTime(System.currentTimeMillis());
1411                     LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1412                     long route_sync_time = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1413                     Thread.sleep(route_sync_time * 1000L);
1414                     setStaleCleanupTime(route_sync_time);
1415                     new RouteCleanup().call();
1416                 } catch (Exception eCancel) {
1417                     LOG.error("Stale Cleanup Task Cancelled", eCancel);
1418                 }
1419             }
1420         };
1421         lastReplayJobFt = executor.submit(task);
1422     }
1423
1424     private boolean previousReplayJobInProgress() {
1425         return lastReplayJobFt != null && !lastReplayJobFt.isDone();
1426     }
1427
1428     private void cancelPreviousReplayJob() {
1429         try {
1430             LOG.error("cancelling already running bgp replay task");
1431             lastReplayJobFt.cancel(true);
1432             lastReplayJobFt = null;
1433             Thread.sleep(2000);
1434         } catch (Throwable e) {
1435             LOG.error("Failed to cancel previous replay job ",e);
1436         }
1437     }
1438
1439     private static void doRouteSync() {
1440         BgpSyncHandle bsh = BgpSyncHandle.getInstance();
1441         LOG.error("Starting BGP route sync");
1442         try {
1443             bgpRouter.initRibSync(bsh);
1444         } catch (Exception e) {
1445             LOG.error("Route sync aborted, exception when initializing: " + e);
1446             return;
1447         }
1448         while (bsh.getState() != bsh.DONE) {
1449             Routes routes = null;
1450             try {
1451                 routes = bgpRouter.doRibSync(bsh);
1452             } catch (Exception e) {
1453                 LOG.error("Route sync aborted, exception when syncing: " + e);
1454                 return;
1455             }
1456             Iterator<Update> updates = routes.getUpdatesIterator();
1457             while (updates.hasNext()) {
1458                 Update u = updates.next();
1459                 Map<String, Map<String, String>> stale_fib_rd_map = BgpConfigurationManager.getStaledFibEntriesMap();
1460                 String rd = u.getRd();
1461                 String nexthop = u.getNexthop();
1462                 int label = u.getLabel();
1463                 String prefix = u.getPrefix();
1464                 int plen = u.getPrefixlen();
1465                 onUpdatePushRoute(rd, prefix, plen, nexthop, label);
1466             }
1467         }
1468         try {
1469             LOG.error("Ending BGP route-sync");
1470             bgpRouter.endRibSync(bsh);
1471         } catch (Exception e) {
1472         }
1473     }
1474
1475     /* onUpdatePushRoute
1476      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1477      *  - Entry compare shall include NextHop, Label.
1478      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1479      *  - If entry not found, add to FIB Config DS.
1480      *  - If entry found, but either Label/NextHop doesn't match.
1481      *      - Update FIB Config DS with modified values.
1482      *      - delete from Stale Map.
1483      */
1484     public static void onUpdatePushRoute(String rd, String prefix, int plen, String nextHop, int label) {
1485         Map<String, Map<String, String>> stale_fib_rd_map = BgpConfigurationManager.getStaledFibEntriesMap();
1486         boolean addroute = false;
1487         if (!stale_fib_rd_map.isEmpty()) {
1488             // restart Scenario, as MAP is not empty.
1489             Map<String, String> map = stale_fib_rd_map.get(rd);
1490             if (map != null) {
1491                 String nexthoplabel = map.get(prefix + "/" + plen);
1492                 if (null == nexthoplabel) {
1493                     // New Entry, which happened to be added during restart.
1494                     addroute = true;
1495                 } else {
1496                     map.remove(prefix + "/" + plen);
1497                     if (isRouteModified(nextHop, label, nexthoplabel)) {
1498                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1499                         // Existing entry, where in Nexthop/Label got modified during restart
1500                         addroute = true;
1501                     }
1502                 }
1503             }
1504         } else {
1505             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1506             addroute = true;
1507         }
1508         if (addroute) {
1509             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1510             fibDSWriter.addFibEntryToDS(rd, prefix + "/" + plen, Arrays.asList(nextHop), label, RouteOrigin.BGP);
1511             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1512         }
1513     }
1514
1515     private static boolean isRouteModified(String nexthop, int label, String nexthoplabel) {
1516         return !nexthoplabel.isEmpty() && !nexthoplabel.equals(nexthop + "/" + label);
1517     }
1518
1519     static private void replayNbrConfig(List<Neighbors> n, BgpRouter br) {
1520         for (Neighbors nbr : n) {
1521             try {
1522                 br.addNeighbor(nbr.getAddress().getValue(),
1523                         nbr.getRemoteAs().intValue());
1524                 //itmProvider.buildTunnelsToDCGW(new IpAddress(nbr.getAddress().getValue().toCharArray()));
1525             } catch (Exception e) {
1526                 LOG.error("Replay:addNbr() received exception: \"" + e + "\"");
1527                 continue;
1528             }
1529             EbgpMultihop en = nbr.getEbgpMultihop();
1530             if (en != null) {
1531                 try {
1532                     br.addEbgpMultihop(en.getPeerIp().getValue(),
1533                             en.getNhops().intValue());
1534                 } catch (Exception e) {
1535                     LOG.error("Replay:addEBgp() received exception: \"" + e + "\"");
1536                 }
1537             }
1538             UpdateSource us = nbr.getUpdateSource();
1539             if (us != null) {
1540                 try {
1541                     br.addUpdateSource(us.getPeerIp().getValue(),
1542                             us.getSourceIp().getValue());
1543                 } catch (Exception e) {
1544                     LOG.error("Replay:addUS() received exception: \"" + e + "\"");
1545                 }
1546             }
1547             List<AddressFamilies> afs = nbr.getAddressFamilies();
1548             if (afs != null) {
1549                 for (AddressFamilies af : afs) {
1550                     af_afi afi = af_afi.findByValue(af.getAfi().intValue());
1551                     af_safi safi = af_safi.findByValue(af.getSafi().intValue());
1552                     try {
1553                         br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
1554                     } catch (Exception e) {
1555                         LOG.error("Replay:addAf() received exception: \"" + e + "\"");
1556                     }
1557                 }
1558             }
1559         }
1560     }
1561
1562     public static String getConfigHost() {
1563         if (config == null) {
1564             return cHostStartup;
1565         }
1566         ConfigServer ts = config.getConfigServer();
1567         return (ts == null ? cHostStartup : ts.getHost().getValue());
1568     }
1569
1570     public static int getConfigPort() {
1571         if (config == null) {
1572             return Integer.parseInt(cPortStartup);
1573         }
1574         ConfigServer ts = config.getConfigServer();
1575         return (ts == null ? Integer.parseInt(cPortStartup) :
1576                 ts.getPort().intValue());
1577     }
1578
1579     public static Bgp getConfig() {
1580         //TODO cleanup this cache code
1581         try {
1582             Optional<Bgp> optional = BgpUtil.read(dataBroker,
1583                     LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(Bgp.class));
1584             return optional.get();
1585         } catch (Exception e) {
1586             //LOG.error("failed to get bgp config",e);
1587         }
1588         return null;
1589     }
1590
1591     public synchronized void replay() {
1592         synchronized (bgpConfigurationManager) {
1593             String host = getConfigHost();
1594             int port = getConfigPort();
1595             LOG.error("connecting  to bgp host {} ", host);
1596
1597             boolean res = bgpRouter.connect(host, port);
1598             if (!res) {
1599                 String msg = "Cannot connect to BGP config server at " + host + ":" + port;
1600                 if (config != null) {
1601                     msg += "; Configuration Replay aborted";
1602                 }
1603                 LOG.error(msg);
1604                 return;
1605             }
1606             config = getConfig();
1607             if (config == null) {
1608                 LOG.error("bgp config is empty nothing to push to bgp");
1609                 return;
1610             }
1611             BgpRouter br = bgpRouter;
1612             AsId a = config.getAsId();
1613             if (a == null) {
1614                 return;
1615             }
1616             int asNum = a.getLocalAs().intValue();
1617             Ipv4Address routerId = a.getRouterId();
1618             Long spt = a.getStalepathTime();
1619             Boolean afb = a.isAnnounceFbit();
1620             String rid = (routerId == null) ? "" : routerId.getValue();
1621             int stalepathTime = (int) getStalePathtime(0, config.getAsId());
1622             boolean announceFbit = (afb == null) ? false : afb.booleanValue();
1623             try {
1624                 br.startBgp(asNum, rid, stalepathTime, announceFbit);
1625             } catch (BgpRouterException bre) {
1626                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
1627                     doRouteSync();
1628                 } else {
1629                     LOG.error("Replay: startBgp() received exception: \""
1630                             + bre + "\"; " + addWarn);
1631                 }
1632             } catch (Exception e) {
1633                 //not unusual. We may have restarted & BGP is already on
1634                 LOG.error("Replay:startBgp() received exception: \"" + e + "\"");
1635             }
1636
1637             if (getBgpCounters() == null) {
1638                 startBgpCountersTask();
1639             }
1640
1641             if (getBgpAlarms() == null) {
1642                 startBgpAlarmsTask();
1643             }
1644
1645             Logging l = config.getLogging();
1646             if (l != null) {
1647                 try {
1648                     br.setLogging(l.getFile(), l.getLevel());
1649                 } catch (Exception e) {
1650                     LOG.error("Replay:setLogging() received exception: \"" + e + "\"");
1651                 }
1652             }
1653
1654             GracefulRestart g = config.getGracefulRestart();
1655             if (g != null) {
1656                 try {
1657                     br.addGracefulRestart(g.getStalepathTime().intValue());
1658                 } catch (Exception e) {
1659                     LOG.error("Replay:addGr() received exception: \"" + e + "\"");
1660                 }
1661             }
1662
1663             List<Neighbors> n = config.getNeighbors();
1664             if (n != null) {
1665                 LOG.error("configuring existing Neighbors present for replay total neighbors {}", n.size());
1666                 replayNbrConfig(n, br);
1667             } else {
1668                 LOG.error("no Neighbors present for replay config ");
1669             }
1670
1671             List<Vrfs> v = config.getVrfs();
1672             if (v != null) {
1673                 for (Vrfs vrf : v) {
1674                     try {
1675                         br.addVrf(vrf.getRd(), vrf.getImportRts(),
1676                                 vrf.getExportRts());
1677                     } catch (Exception e) {
1678                         LOG.error("Replay:addVrf() received exception: \"" + e + "\"");
1679                     }
1680                 }
1681             }
1682
1683             List<Networks> ln = config.getNetworks();
1684             if (ln != null) {
1685                 for (Networks net : ln) {
1686                     String rd = net.getRd();
1687                     String pfxlen = net.getPrefixLen();
1688                     String nh = net.getNexthop().getValue();
1689                     Long label = net.getLabel();
1690                     int lbl = (label == null) ? 0 : label.intValue();
1691                     if (rd == null && lbl > 0) {
1692                         //LU prefix is being deleted.
1693                         rd = Integer.toString(lbl);
1694                     }
1695                     try {
1696                         br.addPrefix(rd, pfxlen, nh, lbl);
1697                     } catch (Exception e) {
1698                         LOG.error("Replay:addPfx() received exception: \"" + e + "\"");
1699                     }
1700                 }
1701             }
1702         }
1703     }
1704
1705     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
1706         BgpUtil.update(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1707     }
1708
1709     private <T extends DataObject> void asyncWrite(InstanceIdentifier<T> iid, T dto) {
1710         BgpUtil.write(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
1711     }
1712
1713     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
1714         BgpUtil.delete(dataBroker, LogicalDatastoreType.CONFIGURATION, iid);
1715     }
1716
1717     public synchronized void
1718     startConfig(String bgpHost, int thriftPort) {
1719         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
1720                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
1721         InstanceIdentifier<ConfigServer> iid = iib.build();
1722         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
1723         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
1724                 .setPort((long) thriftPort).build();
1725         update(iid, dto);
1726     }
1727
1728     public synchronized void
1729     startBgp(int as, String routerId, int spt, boolean fbit) {
1730         Long localAs = (long) as;
1731         Ipv4Address rid = (routerId == null) ?
1732                 null : new Ipv4Address(routerId);
1733         Long staleTime = (long) spt;
1734         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
1735                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
1736         InstanceIdentifier<AsId> iid = iib.build();
1737         AsId dto = new AsIdBuilder().setLocalAs(localAs)
1738                 .setRouterId(rid)
1739                 .setStalepathTime(staleTime)
1740                 .setAnnounceFbit(fbit).build();
1741         update(iid, dto);
1742     }
1743
1744     public synchronized void
1745     addLogging(String fileName, String logLevel) {
1746         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
1747                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
1748         InstanceIdentifier<Logging> iid = iib.build();
1749         Logging dto = new LoggingBuilder().setFile(fileName)
1750                 .setLevel(logLevel).build();
1751         update(iid, dto);
1752     }
1753
1754     public synchronized void
1755     addGracefulRestart(int staleTime) {
1756         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
1757                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
1758         InstanceIdentifier<GracefulRestart> iid = iib.build();
1759         GracefulRestart dto = new GracefulRestartBuilder()
1760                 .setStalepathTime((long) staleTime).build();
1761         update(iid, dto);
1762     }
1763
1764     public synchronized void
1765     addNeighbor(String nbrIp, int remoteAs) {
1766         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1767         Long rAs = (long) remoteAs;
1768         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
1769                 InstanceIdentifier.builder(Bgp.class)
1770                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
1771         InstanceIdentifier<Neighbors> iid = iib.build();
1772         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
1773                 .setRemoteAs(rAs).build();
1774         update(iid, dto);
1775     }
1776
1777     public synchronized void
1778     addUpdateSource(String nbrIp, String srcIp) {
1779         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1780         Ipv4Address srcAddr = new Ipv4Address(srcIp);
1781         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
1782                 InstanceIdentifier.builder(Bgp.class)
1783                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1784                         .child(UpdateSource.class);
1785         InstanceIdentifier<UpdateSource> iid = iib.build();
1786         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
1787                 .setSourceIp(srcAddr).build();
1788         update(iid, dto);
1789     }
1790
1791     public synchronized void
1792     addEbgpMultihop(String nbrIp, int nHops) {
1793         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1794         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
1795                 InstanceIdentifier.builder(Bgp.class)
1796                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1797                         .child(EbgpMultihop.class);
1798         InstanceIdentifier<EbgpMultihop> iid = iib.build();
1799         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
1800                 .setNhops((long) nHops).build();
1801         update(iid, dto);
1802     }
1803
1804     public synchronized void
1805     addAddressFamily(String nbrIp, int afi, int safi) {
1806         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1807         Long a = (long) afi;
1808         Long sa = (long) safi;
1809         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
1810                 InstanceIdentifier.builder(Bgp.class)
1811                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1812                         .child(AddressFamilies.class, new AddressFamiliesKey(a, sa));
1813         InstanceIdentifier<AddressFamilies> iid = iib.build();
1814         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
1815                 .setAfi(a).setSafi(sa).build();
1816         update(iid, dto);
1817     }
1818
1819     public synchronized void
1820     addPrefix(String rd, String pfx, List<String> nhList, int lbl) {
1821         for (String nh : nhList) {
1822             Ipv4Address nexthop = new Ipv4Address(nh);
1823             Long label = (long) lbl;
1824             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
1825                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
1826             Networks dto = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
1827                                                 .setLabel(label).build();
1828             update(iid, dto);
1829         }
1830     }
1831
1832     public synchronized void
1833     addVrf(String rd, List<String> irts, List<String> erts) {
1834         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
1835                 InstanceIdentifier.builder(Bgp.class)
1836                         .child(Vrfs.class, new VrfsKey(rd));
1837         InstanceIdentifier<Vrfs> iid = iib.build();
1838         Vrfs dto = new VrfsBuilder().setRd(rd)
1839                 .setImportRts(irts)
1840                 .setExportRts(erts).build();
1841
1842         asyncWrite(iid, dto);
1843     }
1844
1845     public synchronized void stopConfig() {
1846         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
1847                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
1848         InstanceIdentifier<ConfigServer> iid = iib.build();
1849         delete(iid);
1850     }
1851
1852     public synchronized void stopBgp() {
1853         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
1854                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
1855         InstanceIdentifier<AsId> iid = iib.build();
1856         delete(iid);
1857     }
1858
1859     public synchronized void delLogging() {
1860         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
1861                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
1862         InstanceIdentifier<Logging> iid = iib.build();
1863         delete(iid);
1864     }
1865
1866     public synchronized void delGracefulRestart() {
1867         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
1868                 InstanceIdentifier.builder(Bgp.class)
1869                         .child(GracefulRestart.class);
1870         InstanceIdentifier<GracefulRestart> iid = iib.build();
1871         delete(iid);
1872     }
1873
1874     public synchronized void delNeighbor(String nbrIp) {
1875         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1876         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
1877                 InstanceIdentifier.builder(Bgp.class)
1878                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
1879         InstanceIdentifier<Neighbors> iid = iib.build();
1880         delete(iid);
1881     }
1882
1883     public synchronized void delUpdateSource(String nbrIp) {
1884         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1885         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
1886                 InstanceIdentifier.builder(Bgp.class)
1887                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1888                         .child(UpdateSource.class);
1889         InstanceIdentifier<UpdateSource> iid = iib.build();
1890         delete(iid);
1891     }
1892
1893     public synchronized void delEbgpMultihop(String nbrIp) {
1894         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1895         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
1896                 InstanceIdentifier.builder(Bgp.class)
1897                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1898                         .child(EbgpMultihop.class);
1899         InstanceIdentifier<EbgpMultihop> iid = iib.build();
1900         delete(iid);
1901     }
1902
1903     public synchronized void
1904     delAddressFamily(String nbrIp, int afi, int safi) {
1905         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
1906         Long a = (long) afi;
1907         Long sa = (long) safi;
1908         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
1909                 InstanceIdentifier.builder(Bgp.class)
1910                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
1911                         .child(AddressFamilies.class, new AddressFamiliesKey(a, sa));
1912         InstanceIdentifier<AddressFamilies> iid = iib.build();
1913         delete(iid);
1914     }
1915
1916     public synchronized void delPrefix(String rd, String pfx) {
1917         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
1918                 InstanceIdentifier.builder(Bgp.class)
1919                         .child(Networks.class, new NetworksKey(pfx, rd));
1920         InstanceIdentifier<Networks> iid = iib.build();
1921         delete(iid);
1922     }
1923
1924     public synchronized void delVrf(String rd) {
1925         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
1926                 InstanceIdentifier.builder(Bgp.class)
1927                         .child(Vrfs.class, new VrfsKey(rd));
1928         InstanceIdentifier<Vrfs> iid = iib.build();
1929         delete(iid);
1930     }
1931
1932     static ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
1933
1934     /*
1935     * Remove Stale Marked Routes after timer expiry.
1936     */
1937     class RouteCleanup implements Callable<Integer> {
1938
1939         @Override
1940         public Integer call() {
1941             totalCleared = 0;
1942             try {
1943                 if (staledFibEntriesMap.isEmpty()) {
1944                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
1945                 } else {
1946                     for (String rd : staledFibEntriesMap.keySet()) {
1947                         if (Thread.interrupted()) {
1948                             return 0;
1949                         }
1950                         Map<String, String> map = staledFibEntriesMap.get(rd);
1951                         if (map != null) {
1952                             for (String prefix : map.keySet()) {
1953                                 if (Thread.interrupted()) {
1954                                     return 0;
1955                                 }
1956                                 try {
1957                                     totalCleared++;
1958                                     LOG.error("BGP: RouteCleanup deletePrefix called but not executed rd:{}, prefix{}" + rd.toString() + prefix);
1959                                     // fibDSWriter.removeFibEntryFromDS(rd, prefix);
1960                                 } catch (Exception e) {
1961                                     LOG.error("BGP: RouteCleanup deletePrefix failed rd:{}, prefix{}" + rd.toString() + prefix);
1962                                 }
1963                             }
1964                         }
1965                     }
1966                 }
1967             } catch (Exception e) {
1968                 LOG.error("Cleanup Thread Got interrupted, Failed to cleanup stale routes ", e);
1969             } finally {
1970                 staledFibEntriesMap.clear();
1971             }
1972             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
1973             return 0;
1974         }
1975     }
1976
1977     /*
1978      * BGP restart scenario, ODL-BGP manager was/is running.
1979      * On re-sync notification, Get a copy of FIB database.
1980      */
1981     public static void createStaleFibMap() {
1982         totalStaledCount = 0;
1983         try {
1984             /*
1985             * at the time Stale FIB creation, Wait till all PENDING write transaction
1986              * to complete (or)wait for max timeout value of STALE_FIB_WAIT Seconds.
1987              */
1988             int retry = STALE_FIB_WAIT;
1989             while ((BgpUtil.getGetPendingWrTransaction() != 0) && (retry > 0)) {
1990                 Thread.sleep(1000);
1991                 retry--;
1992                 if (retry == 0) {
1993                     LOG.error("TimeOut occured {} seconds, in waiting stale fibDSWriter create", STALE_FIB_WAIT);
1994                 }
1995             }
1996             staledFibEntriesMap.clear();
1997             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
1998             DataBroker db = BgpUtil.getBroker();
1999             if (db == null) {
2000                 LOG.error("Couldn't find BgpUtil dataBroker while creating createStaleFibMap");
2001                 return;
2002             }
2003
2004             Optional<FibEntries> fibEntries = BgpUtil.read(BgpUtil.getBroker(),
2005                     LogicalDatastoreType.CONFIGURATION, id);
2006             if (fibEntries.isPresent()) {
2007                 List<VrfTables> stale_vrfTables = fibEntries.get().getVrfTables();
2008                 for (VrfTables vrfTable : stale_vrfTables) {
2009                     Map<String, String> stale_fib_ent_map = new HashMap<>();
2010                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2011                         if (Thread.interrupted()) {
2012                             break;
2013                         }
2014                         totalStaledCount++;
2015                         //Create MAP from stale_vrfTables.
2016                         for (String nextHop : vrfEntry.getNextHopAddressList()) {
2017                             stale_fib_ent_map.put(vrfEntry.getDestPrefix(), nextHop + "/" + vrfEntry.getLabel());
2018                         }
2019                     }
2020                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), stale_fib_ent_map);
2021                 }
2022             } else {
2023                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
2024             }
2025         } catch (Exception e) {
2026             LOG.error("createStaleFibMap:: error ", e);
2027         }
2028         LOG.error("created {} staled entries ", totalStaledCount);
2029     }
2030
2031     //map<rd, map<prefix/len, nexthop/label>>
2032     public static Map<String, Map<String, String>> getStaledFibEntriesMap() {
2033         return staledFibEntriesMap;
2034     }
2035
2036     //TODO: below function is for testing purpose with cli
2037     public static void onUpdateWithdrawRoute(String rd, String prefix, int plen) {
2038         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
2039         try {
2040             fibDSWriter.removeFibEntryFromDS(rd, prefix + "/" + plen);
2041         } catch (Throwable e) {
2042             LOG.error("failed to handle withdraw route ", e);
2043         }
2044     }
2045     public boolean isBgpConnected(){
2046         return bgpRouter.isBgpConnected();
2047     }
2048     public long getLastConnectedTS() {
2049         return bgpRouter.getLastConnectedTS();
2050     }
2051     public long getConnectTS() {
2052         return bgpRouter.getConnectTS();
2053     }
2054     public long getStartTS() {
2055         return bgpRouter.getStartTS();
2056     }
2057     public static int getTotalStaledCount() {return totalStaledCount;}
2058     public static int getTotalCleared() { return totalCleared;}
2059
2060     public Timer getBgpCountersTimer() {
2061         return bgpCountersTimer;
2062     }
2063
2064     public BgpCounters getBgpCounters() {
2065         return bgpCounters;
2066     }
2067
2068     public  void setBgpCountersTimer (Timer t) {
2069         bgpCountersTimer = t;
2070     }
2071     public  void setBgpAlarmsTimer (Timer t) {
2072         bgpAlarmsTimer = t;
2073     }
2074
2075     public void startBgpCountersTask() {
2076         if (getBgpCounters() == null) {
2077
2078             try {
2079                 bgpCounters = new BgpCounters(bgpConfigurationManager.getBgpSdncMipIp());
2080                 setBgpCountersTimer(new Timer(true));
2081                 getBgpCountersTimer().scheduleAtFixedRate(bgpCounters, 0, 120 * 1000);
2082
2083
2084                 LOG.info("Bgp Counters task scheduled for every two minutes.");
2085             } catch (Exception e) {
2086                 System.out.println("Could not start the timertask for Bgp Counters.");
2087                 e.printStackTrace();
2088             }
2089
2090             try {
2091                 bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
2092             } catch (Exception e) {
2093                 System.out.println("Could not set the default options for logging");
2094             }
2095         }
2096     }
2097
2098     public void stopBgpCountersTask() {
2099         Timer t = getBgpCountersTimer();
2100         if (getBgpCounters() != null) {
2101             t.cancel();
2102             setBgpCountersTimer(null);
2103             bgpCounters = null;
2104         }
2105     }
2106     public void startBgpAlarmsTask() {
2107         if (getBgpAlarms() == null) {
2108             try {
2109                 bgpAlarms = new BgpAlarms(this);
2110                 setBgpAlarmsTimer(new Timer(true));
2111                 getBgpAlarmsTimer().scheduleAtFixedRate(bgpAlarms, 0, 60 * 1000);
2112                 LOG.info("Bgp Alarms task scheduled for every minute.");
2113             } catch (Exception e) {
2114                 System.out.println("Could not start the timertask for Bgp Alarms.");
2115                 e.printStackTrace();
2116             }
2117         }
2118     }
2119
2120     public void stopBgpAlarmsTask() {
2121         Timer t = getBgpAlarmsTimer();
2122         if (getBgpAlarms() != null) {
2123             t.cancel();
2124             setBgpAlarmsTimer(null);
2125             bgpAlarms = null;
2126         }
2127     }
2128     public Timer getBgpAlarmsTimer() {
2129         return bgpAlarmsTimer;
2130     }
2131
2132     public BgpAlarms getBgpAlarms() {
2133         return bgpAlarms;
2134     }
2135
2136 }