bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.util.concurrent.ThreadFactoryBuilder;
  12 import io.netty.util.concurrent.GlobalEventExecutor;
  13 import java.io.IOException;
  14 import java.lang.reflect.Constructor;
  15 import java.lang.reflect.InvocationTargetException;
  16 import java.net.InetAddress;
  17 import java.net.NetworkInterface;
  18 import java.net.SocketException;
  19 import java.util.ArrayList;
  20 import java.util.Collection;
  21 import java.util.Collections;
  22 import java.util.Enumeration;
  23 import java.util.HashMap;
  24 import java.util.Iterator;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Timer;
  29 import java.util.concurrent.Callable;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.CountDownLatch;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.Executors;
  34 import java.util.concurrent.Future;
  35 import java.util.concurrent.ScheduledExecutorService;
  36 import java.util.concurrent.ScheduledFuture;
  37 import java.util.concurrent.TimeUnit;
  38 import java.util.concurrent.TimeoutException;
  39 import java.util.concurrent.atomic.AtomicInteger;
  40 import java.util.concurrent.atomic.AtomicReference;
  41 import javax.annotation.Nullable;
  42 import javax.annotation.PreDestroy;
  43 import javax.inject.Inject;
  44 import javax.inject.Singleton;
  45 import org.apache.thrift.TApplicationException;
  46 import org.apache.thrift.TException;
  47 import org.apache.thrift.transport.TTransport;
  48 import org.opendaylight.controller.config.api.osgi.WaitingServiceTracker;
  49 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  50 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  51 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  52 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  53 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  54 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  55 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  56 import org.opendaylight.genius.mdsalutil.NwConstants;
  57 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  58 import org.opendaylight.infrautils.metrics.MetricProvider;
  59 import org.opendaylight.mdsal.eos.binding.api.Entity;
  60 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipCandidateRegistration;
  61 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipListenerRegistration;
  62 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  63 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  64 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  65 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  66 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  67 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  68 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  69 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  70 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  71 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  72 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  73 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  74 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  75 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  76 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  77 import org.opendaylight.netvirt.bgpmanager.thrift.gen.protocol_type;
  78 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  79 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  80 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  81 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  82 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.AddressFamily;
  83 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  84 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
  85 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EncapType;
  86 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.TcpMd5SignaturePasswordType;
  87 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
  88 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
  90 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
  92 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
  94 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Multipath;
  96 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathKey;
  98 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Neighbors;
  99 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsKey;
 101 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Networks;
 102 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksKey;
 104 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpath;
 105 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathKey;
 107 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Vrfs;
 108 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsKey;
 110 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamilies;
 111 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesKey;
 113 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihop;
 114 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihopBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSource;
 116 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSourceBuilder;
 117 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfs.AddressFamiliesVrf;
 118 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfs.AddressFamiliesVrfBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.TcpSecurityOption;
 120 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOption;
 121 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOptionBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
 123 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.macvrfentries.MacVrfEntry;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 129 import org.opendaylight.yangtools.yang.binding.DataObject;
 130 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 131 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 132 import org.osgi.framework.BundleContext;
 133 import org.slf4j.Logger;
 134 import org.slf4j.LoggerFactory;
 135
 136 @Singleton
 137 public class BgpConfigurationManager {
 138     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 139
 140     // to have stale FIB map (RD, Prefix)
 141     //  number of seconds wait for route sync-up between ODL and BGP
 142     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 143     private static final String DEF_LOGFILE = "/var/log/bgp_debug.log";
 144     private static final String DEF_LOGLEVEL = "errors";
 145     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 146     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 147     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 148     private static final String DEF_UPORT = "6644";
 149     private static final String DEF_CHOST = "255.255.255.255"; // Invalid Host IP
 150     private static final String DEF_CPORT = "0";               // Invalid Port
 151     private static final String DEF_BGP_SDNC_MIP = "127.0.0.1";
 152     private static final String BGP_SDNC_MIP = "vpnservice.bgp.thrift.sdnc.mip";
 153     private static final int RESTART_DEFAULT_GR = 90;
 154     private static final int DS_RETRY_COUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
 155     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
 156     private static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 157     private static final String BGP_ENTITY_NAME = "bgp";
 158     private static final String ADD_WARN = "Config store updated; undo with Delete if needed.";
 159     private static final String DEL_WARN = "Config store updated; undo with Add if needed.";
 160     private static final String UPD_WARN = "Update operation not supported; Config store updated;"
 161             + " restore with another Update if needed.";
 162
 163     private static final Class<?>[] REACTORS = {
 164         ConfigServerReactor.class, AsIdReactor.class,
 165         GracefulRestartReactor.class, LoggingReactor.class,
 166         NeighborsReactor.class, UpdateSourceReactor.class,
 167         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 168         NetworksReactor.class, VrfsReactor.class, BgpReactor.class,
 169         MultipathReactor.class, VrfMaxpathReactor.class
 170     };
 171
 172     private IBgpManager bgpManager;
 173     private final DataBroker dataBroker;
 174     private final FibDSWriter fibDSWriter;
 175     private final IVpnLinkService vpnLinkService;
 176     private final BundleContext bundleContext;
 177     private final BgpUtil bgpUtil;
 178     private volatile Bgp config;
 179     private final BgpRouter bgpRouter;
 180     private final BgpSyncHandle bgpSyncHandle = new BgpSyncHandle();
 181     private BgpThriftService updateServer;
 182
 183     private final CountDownLatch initer = new CountDownLatch(1);
 184
 185     private final String hostStartup;
 186     private final String portStartup;
 187
 188     private final AtomicReference<BgpCounters> bgpCountersReference = new AtomicReference<>();
 189     private ScheduledFuture<?> bgpCountersTask;
 190
 191     private final AtomicReference<BgpAlarms> bgpAlarmsReference = new AtomicReference<>();
 192     private ScheduledFuture<?> bgpAlarmsTask;
 193
 194     private Future<?> lastReplayJobFt;
 195     private ScheduledFuture<?> routeCleanupFuture;
 196
 197     private long staleStartTime;
 198     private long staleEndTime;
 199     private long cfgReplayStartTime;
 200     private long cfgReplayEndTime;
 201     private long staleCleanupTime;
 202     private int totalStaledCount;
 203     private int totalCleared;
 204     private int totalExternalRoutes;
 205     private int totalExternalMacRoutes;
 206
 207     private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(
 208             new ThreadFactoryBuilder().setNameFormat("BgpConfigurationManager-%d").setDaemon(true).build());
 209
 210     /**
 211      * this map store the new address families to send to quagga. When it is sended you must clear it.
 212      * The keys String are rd (route distinguisher).
 213      * */
 214     private final ConcurrentHashMap<String, List<AddressFamiliesVrf>> mapNewAdFamily = new ConcurrentHashMap<>();
 215
 216     // map<rd, map<prefix/len:nexthop, label>>
 217     private final Map<String, Map<String, Long>> staledFibEntriesMap = new ConcurrentHashMap<>();
 218
 219     // map<rd, map<tep-ip, map<mac, l2vni>>>
 220     private final Map<String, Map<String, Map<String, Long>>> rt2TepMap = new ConcurrentHashMap<>();
 221
 222     private final List<AutoCloseable> listeners = new ArrayList<>();
 223
 224     private final EntityOwnershipUtils entityOwnershipUtils;
 225     private final EntityOwnershipCandidateRegistration candidateRegistration;
 226     private final EntityOwnershipListenerRegistration entityListenerRegistration;
 227     private final MetricProvider metricProvider;
 228
 229     @Inject
 230     public BgpConfigurationManager(final DataBroker dataBroker,
 231             final EntityOwnershipService entityOwnershipService,
 232             final FibDSWriter fibDSWriter,
 233             final IVpnLinkService vpnLinkSrvce,
 234             final BundleContext bundleContext,
 235             final BgpUtil bgpUtil,
 236             final MetricProvider metricProvider) {
 237         this.dataBroker = dataBroker;
 238         this.fibDSWriter = fibDSWriter;
 239         this.vpnLinkService = vpnLinkSrvce;
 240         this.bundleContext = bundleContext;
 241         this.bgpUtil = bgpUtil;
 242         this.metricProvider = metricProvider;
 243         String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
 244         hostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 245         portStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 246         LOG.info("UpdateServer at localhost:{}, ConfigServer at {}:{}", updatePort, hostStartup, portStartup);
 247         VtyshCli.setHostAddr(hostStartup);
 248         ClearBgpCli.setHostAddr(hostStartup);
 249         bgpRouter = BgpRouter.newInstance(this::getConfig, this::isBGPEntityOwner);
 250         registerCallbacks();
 251
 252         entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 253
 254         candidateRegistration = registerEntityCandidate(entityOwnershipService);
 255         entityListenerRegistration = registerEntityListener(entityOwnershipService);
 256
 257         LOG.info("BGP Configuration manager initialized");
 258         initer.countDown();
 259
 260         GlobalEventExecutor.INSTANCE.execute(() -> {
 261             final WaitingServiceTracker<IBgpManager> tracker = WaitingServiceTracker.create(
 262                     IBgpManager.class, bundleContext);
 263             bgpManager = tracker.waitForService(WaitingServiceTracker.FIVE_MINUTES);
 264             updateServer = new BgpThriftService(Integer.parseInt(updatePort), bgpManager, this);
 265             updateServer.start();
 266             LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 267         });
 268     }
 269
 270     public String getBgpSdncMipIp() {
 271         return getProperty(BGP_SDNC_MIP, DEF_BGP_SDNC_MIP);
 272     }
 273
 274     public long getStaleCleanupTime() {
 275         return staleCleanupTime;
 276     }
 277
 278     public void setStaleCleanupTime(long staleCleanupTime) {
 279         this.staleCleanupTime = staleCleanupTime;
 280     }
 281
 282     public long getCfgReplayEndTime() {
 283         return cfgReplayEndTime;
 284     }
 285
 286     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 287         this.cfgReplayEndTime = cfgReplayEndTime;
 288     }
 289
 290     public long getCfgReplayStartTime() {
 291         return cfgReplayStartTime;
 292     }
 293
 294     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 295         this.cfgReplayStartTime = cfgReplayStartTime;
 296     }
 297
 298     public long getStaleEndTime() {
 299         return staleEndTime;
 300     }
 301
 302     public void setStaleEndTime(long staleEndTime) {
 303         this.staleEndTime = staleEndTime;
 304     }
 305
 306     public long getStaleStartTime() {
 307         return staleStartTime;
 308     }
 309
 310     public void setStaleStartTime(long staleStartTime) {
 311         this.staleStartTime = staleStartTime;
 312     }
 313
 314     private Object createListener(Class<?> cls) {
 315         try {
 316             Constructor<?> ctor = cls.getConstructor(BgpConfigurationManager.class);
 317             return ctor.newInstance(this);
 318         } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException
 319                 e) {
 320             LOG.error("Failed to create listener object", e);
 321             return null;
 322         }
 323     }
 324
 325     private void registerCallbacks() {
 326         for (Class<?> reactor : REACTORS) {
 327             Object obj = createListener(reactor);
 328             if (obj != null) {
 329                 AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 330                 dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 331                 listeners.add(dcl);
 332             }
 333         }
 334     }
 335
 336     @SuppressWarnings("checkstyle:IllegalCatch")
 337     @PreDestroy
 338     public void close() {
 339         executor.shutdown();
 340
 341         if (updateServer != null) {
 342             updateServer.stop();
 343         }
 344
 345         if (candidateRegistration != null) {
 346             candidateRegistration.close();
 347         }
 348
 349         entityListenerRegistration.close();
 350
 351         listeners.forEach(l -> {
 352             try {
 353                 l.close();
 354             } catch (Exception e) {
 355                 LOG.warn("Error closing {}", l ,e);
 356             }
 357         });
 358
 359         LOG.info("{} close", getClass().getSimpleName());
 360     }
 361
 362     private String getProperty(String var, String def) {
 363         String property = bundleContext.getProperty(var);
 364         return property == null ? def : property;
 365     }
 366
 367     private EntityOwnershipCandidateRegistration registerEntityCandidate(
 368             final EntityOwnershipService entityOwnershipService) {
 369         try {
 370             return entityOwnershipService.registerCandidate(
 371                     new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME));
 372         } catch (CandidateAlreadyRegisteredException e) {
 373             LOG.error("failed to register bgp entity", e);
 374             return null;
 375         }
 376     }
 377
 378     private EntityOwnershipListenerRegistration registerEntityListener(
 379             final EntityOwnershipService entityOwnershipService) {
 380         return entityOwnershipService.registerListener(BGP_ENTITY_TYPE_FOR_OWNERSHIP, ownershipChange -> {
 381             LOG.trace("entity owner change event fired: {}", ownershipChange);
 382
 383             if (ownershipChange.getState() == EntityOwnershipChangeState.LOCAL_OWNERSHIP_GRANTED) {
 384                 LOG.trace("This PL is the Owner");
 385                 activateMIP();
 386                 bgpRestarted();
 387             } else {
 388                 LOG.debug("Not owner: hasOwner: {}, isOwner: {}", ownershipChange.getState().hasOwner(),
 389                         ownershipChange.getState().isOwner());
 390             }
 391         });
 392     }
 393
 394     public boolean isBGPEntityOwner() {
 395         return entityOwnershipUtils.isEntityOwner(new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME), 0, 1);
 396     }
 397
 398     public Bgp get() {
 399         config = getConfig();
 400         return config;
 401     }
 402
 403     public class ConfigServerReactor
 404             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 405             implements ClusteredDataTreeChangeListener<ConfigServer> {
 406         private static final String YANG_OBJ = "config-server ";
 407
 408         public ConfigServerReactor() {
 409             super(ConfigServer.class, ConfigServerReactor.class);
 410         }
 411
 412         @Override
 413         protected void add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 414             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 415             if (!isBGPEntityOwner()) {
 416                 return;
 417             }
 418
 419             try {
 420                 initer.await();
 421             } catch (InterruptedException e) {
 422                 // Ignored
 423             }
 424             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 425             bgpRouter.configServerUpdated();
 426             synchronized (BgpConfigurationManager.this) {
 427                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 428                         val.getPort().intValue());
 429                 if (!res) {
 430                     LOG.error(YANG_OBJ + "Add failed; " + ADD_WARN);
 431                 }
 432             }
 433             VtyshCli.setHostAddr(val.getHost().getValue());
 434             ClearBgpCli.setHostAddr(val.getHost().getValue());
 435         }
 436
 437         @Override
 438         protected ConfigServerReactor getDataTreeChangeListener() {
 439             return ConfigServerReactor.this;
 440         }
 441
 442         @Override
 443         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 444             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 445         }
 446
 447         @Override
 448         protected void remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 449             LOG.trace("received bgp disconnect");
 450             if (!isBGPEntityOwner()) {
 451                 return;
 452             }
 453
 454             bgpRouter.configServerUpdated();
 455
 456             synchronized (BgpConfigurationManager.this) {
 457                 bgpRouter.disconnect();
 458             }
 459         }
 460
 461         @Override
 462         protected void update(InstanceIdentifier<ConfigServer> iid,
 463                 ConfigServer oldval, ConfigServer newval) {
 464             LOG.trace("received bgp Connection update");
 465             if (!isBGPEntityOwner()) {
 466                 return;
 467             }
 468             LOG.error(YANG_OBJ + UPD_WARN);
 469         }
 470     }
 471
 472     private BgpRouter getClient(String yangObj) {
 473         if (bgpRouter == null || !bgpRouter.isBgpConnected()) {
 474             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 475             return null;
 476         }
 477         return bgpRouter;
 478     }
 479
 480     public class AsIdReactor
 481             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 482             implements ClusteredDataTreeChangeListener<AsId> {
 483
 484         private static final String YANG_OBJ = "as-id ";
 485
 486         public AsIdReactor() {
 487             super(AsId.class, AsIdReactor.class);
 488         }
 489
 490         @Override
 491         protected void add(InstanceIdentifier<AsId> iid, AsId val) {
 492             LOG.error("received bgp add asid {}", val);
 493             if (!isBGPEntityOwner()) {
 494                 return;
 495             }
 496             LOG.debug("received add router config asNum {}", val.getLocalAs());
 497             synchronized (BgpConfigurationManager.this) {
 498                 BgpRouter br = getClient(YANG_OBJ);
 499                 if (br == null) {
 500                     LOG.debug("{} Unable to process add for asNum {}; {} {}", YANG_OBJ, val.getLocalAs(),
 501                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 502                     return;
 503                 }
 504                 bgpRestarted();
 505
 506                 startBgpCountersTask();
 507                 startBgpAlarmsTask();
 508             }
 509         }
 510
 511         @Override
 512         protected AsIdReactor getDataTreeChangeListener() {
 513             return AsIdReactor.this;
 514         }
 515
 516         @Override
 517         protected InstanceIdentifier<AsId> getWildCardPath() {
 518             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 519         }
 520
 521         @Override
 522         protected void remove(InstanceIdentifier<AsId> iid, AsId val) {
 523             LOG.error("received delete router config asNum {}", val.getLocalAs());
 524             if (!isBGPEntityOwner()) {
 525                 return;
 526             }
 527             synchronized (BgpConfigurationManager.this) {
 528                 long asNum = val.getLocalAs();
 529                 BgpRouter br = getClient(YANG_OBJ);
 530                 if (br == null) {
 531                     LOG.debug("{} Unable to process remove for asNum {}; {} {}", YANG_OBJ, asNum,
 532                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 533                     return;
 534                 }
 535                 try {
 536                     br.stopBgp(asNum);
 537                 } catch (TException | BgpRouterException e) {
 538                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 539                 }
 540
 541                 stopBgpCountersTask();
 542                 stopBgpAlarmsTask();
 543
 544                 Bgp conf = getConfig();
 545                 if (conf == null) {
 546                     LOG.error("Config Null while removing the as-id");
 547                     return;
 548                 }
 549                 LOG.debug("Removing external routes from FIB");
 550                 deleteExternalFibRoutes();
 551                 List<Neighbors> nbrs = conf.getNeighbors();
 552                 if (nbrs != null && nbrs.size() > 0) {
 553                     LOG.error("Tring to remove the as-id when neighbor config is already present");
 554                     for (Neighbors nbr : nbrs) {
 555                         LOG.debug("Removing Neighbor {} from Data store", nbr.getAddress().getValue());
 556                         delNeighbor(nbr.getAddress().getValue());
 557                     }
 558                 }
 559             }
 560         }
 561
 562         @Override
 563         protected void update(InstanceIdentifier<AsId> iid,
 564                 AsId oldval, AsId newval) {
 565             if (!isBGPEntityOwner()) {
 566                 return;
 567             }
 568             LOG.error(YANG_OBJ + UPD_WARN);
 569         }
 570     }
 571
 572     public class GracefulRestartReactor
 573             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 574             implements ClusteredDataTreeChangeListener<GracefulRestart> {
 575
 576         private static final String YANG_OBJ = "graceful-restart ";
 577
 578         public GracefulRestartReactor() {
 579             super(GracefulRestart.class, GracefulRestartReactor.class);
 580         }
 581
 582         @Override
 583         protected void add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 584             if (!isBGPEntityOwner()) {
 585                 return;
 586             }
 587             synchronized (BgpConfigurationManager.this) {
 588                 int stalePathTime = val.getStalepathTime().intValue();
 589                 BgpRouter br = getClient(YANG_OBJ);
 590                 if (br == null) {
 591                     LOG.error("{} Unable to add stale-path time {}; {} {}", YANG_OBJ, stalePathTime,
 592                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 593                     return;
 594                 }
 595                 try {
 596                     br.addGracefulRestart(stalePathTime);
 597                 } catch (TException | BgpRouterException e) {
 598                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 599                 }
 600             }
 601         }
 602
 603         @Override
 604         protected GracefulRestartReactor getDataTreeChangeListener() {
 605             return GracefulRestartReactor.this;
 606         }
 607
 608         @Override
 609         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 610             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 611         }
 612
 613         @Override
 614         protected void remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 615             if (!isBGPEntityOwner()) {
 616                 return;
 617             }
 618             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 619             synchronized (BgpConfigurationManager.this) {
 620                 BgpRouter br = getClient(YANG_OBJ);
 621                 if (br == null) {
 622                     LOG.error("{} Unable to delete stale-path time; {} {}", YANG_OBJ,
 623                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 624                     return;
 625                 }
 626                 try {
 627                     br.delGracefulRestart();
 628                 } catch (TException | BgpRouterException e) {
 629                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 630                 }
 631             }
 632         }
 633
 634         @Override
 635         protected void update(InstanceIdentifier<GracefulRestart> iid,
 636                 GracefulRestart oldval, GracefulRestart newval) {
 637             if (!isBGPEntityOwner()) {
 638                 return;
 639             }
 640             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 641             synchronized (BgpConfigurationManager.this) {
 642                 int stalePathTime = newval.getStalepathTime().intValue();
 643                 BgpRouter br = getClient(YANG_OBJ);
 644                 if (br == null) {
 645                     LOG.error("{} Unable to update stale-path time to {}; {} {}", YANG_OBJ, stalePathTime,
 646                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 647                     return;
 648                 }
 649                 try {
 650                     br.addGracefulRestart(stalePathTime);
 651                 } catch (TException | BgpRouterException e) {
 652                     LOG.error("{} update received exception; {}", YANG_OBJ, ADD_WARN, e);
 653                 }
 654             }
 655         }
 656     }
 657
 658     public class LoggingReactor
 659             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 660             implements ClusteredDataTreeChangeListener<Logging> {
 661
 662         private static final String YANG_OBJ = "logging ";
 663
 664         public LoggingReactor() {
 665             super(Logging.class, LoggingReactor.class);
 666         }
 667
 668         @Override
 669         protected void add(InstanceIdentifier<Logging> iid, Logging val) {
 670             if (!isBGPEntityOwner()) {
 671                 return;
 672             }
 673             synchronized (BgpConfigurationManager.this) {
 674                 BgpRouter br = getClient(YANG_OBJ);
 675                 if (br == null) {
 676                     LOG.error("{} Unable to add logging for qbgp; {} {}", YANG_OBJ,
 677                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 678                     return;
 679                 }
 680                 try {
 681                     br.setLogging(val.getFile(), val.getLevel());
 682                 } catch (TException | BgpRouterException e) {
 683                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 684                 }
 685             }
 686         }
 687
 688         @Override
 689         protected LoggingReactor getDataTreeChangeListener() {
 690             return LoggingReactor.this;
 691         }
 692
 693         @Override
 694         protected InstanceIdentifier<Logging> getWildCardPath() {
 695             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 696         }
 697
 698         @Override
 699         protected void remove(InstanceIdentifier<Logging> iid, Logging val) {
 700             if (!isBGPEntityOwner()) {
 701                 return;
 702             }
 703             LOG.debug("received remove Logging config val {}", val.getLevel());
 704             synchronized (BgpConfigurationManager.this) {
 705                 BgpRouter br = getClient(YANG_OBJ);
 706                 if (br == null) {
 707                     LOG.error("{} Unable to remove logging for qbgp; {} {}", YANG_OBJ,
 708                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 709                     return;
 710                 }
 711                 try {
 712                     br.setLogging(DEF_LOGFILE, DEF_LOGLEVEL);
 713                 } catch (TException | BgpRouterException e) {
 714                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 715                 }
 716             }
 717         }
 718
 719         @Override
 720         protected void update(InstanceIdentifier<Logging> iid,
 721                 Logging oldval, Logging newval) {
 722             if (!isBGPEntityOwner()) {
 723                 return;
 724             }
 725             synchronized (BgpConfigurationManager.this) {
 726                 BgpRouter br = getClient(YANG_OBJ);
 727                 if (br == null) {
 728                     LOG.error("{} Unable to update logging for qbgp; {} {}", YANG_OBJ,
 729                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 730                     return;
 731                 }
 732                 try {
 733                     br.setLogging(newval.getFile(), newval.getLevel());
 734                 } catch (TException | BgpRouterException e) {
 735                     LOG.error("{} newval received exception; {}", YANG_OBJ, ADD_WARN, e);
 736                 }
 737             }
 738         }
 739     }
 740
 741     public class NeighborsReactor
 742             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 743             implements ClusteredDataTreeChangeListener<Neighbors> {
 744
 745         private static final String YANG_OBJ = "neighbors ";
 746
 747         public NeighborsReactor() {
 748             super(Neighbors.class, NeighborsReactor.class);
 749         }
 750
 751         @Override
 752         protected void add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 753             if (!isBGPEntityOwner()) {
 754                 return;
 755             }
 756             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 757             synchronized (BgpConfigurationManager.this) {
 758                 String peerIp = val.getAddress().getValue();
 759                 long as = val.getRemoteAs();
 760                 final String md5Secret = extractMd5Secret(val);
 761                 BgpRouter br = getClient(YANG_OBJ);
 762                 if (br == null) {
 763                     LOG.debug("{} Unable to process add for peer {} as {}; {} {}", YANG_OBJ, peerIp, as,
 764                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 765                     return;
 766                 }
 767                 try {
 768                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 769                     br.addNeighbor(peerIp, as, md5Secret);
 770
 771                 } catch (TException | BgpRouterException e) {
 772                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 773                 }
 774             }
 775         }
 776
 777         @Override
 778         protected NeighborsReactor getDataTreeChangeListener() {
 779             return NeighborsReactor.this;
 780         }
 781
 782         @Override
 783         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 784             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class);
 785         }
 786
 787         @Override
 788         protected void remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 789             if (!isBGPEntityOwner()) {
 790                 return;
 791             }
 792             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 793             synchronized (BgpConfigurationManager.this) {
 794                 String peerIp = val.getAddress().getValue();
 795                 BgpRouter br = getClient(YANG_OBJ);
 796                 if (br == null) {
 797                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 798                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 799                     return;
 800                 }
 801                 try {
 802                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 803                     br.delNeighbor(peerIp);
 804                 } catch (TException | BgpRouterException e) {
 805                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 806                 }
 807
 808                 final BgpAlarms bgpAlarms = getBgpAlarms();
 809                 if (bgpAlarms != null) {
 810                     bgpAlarms.clearBgpNbrDownAlarm(peerIp);
 811                 }
 812             }
 813         }
 814
 815         @Override
 816         protected void update(InstanceIdentifier<Neighbors> iid,
 817                 Neighbors oldval, Neighbors newval) {
 818             if (!isBGPEntityOwner()) {
 819                 return;
 820             }
 821             //purposefully nothing to do.
 822         }
 823     }
 824
 825     public class EbgpMultihopReactor
 826             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 827             implements ClusteredDataTreeChangeListener<EbgpMultihop> {
 828
 829         private static final String YANG_OBJ = "ebgp-multihop ";
 830
 831         public EbgpMultihopReactor() {
 832             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 833         }
 834
 835         @Override
 836         protected void add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 837             if (!isBGPEntityOwner()) {
 838                 return;
 839             }
 840             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 841             synchronized (BgpConfigurationManager.this) {
 842                 String peerIp = val.getPeerIp().getValue();
 843                 BgpRouter br = getClient(YANG_OBJ);
 844                 if (br == null) {
 845                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
 846                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 847                     return;
 848                 }
 849                 try {
 850                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 851                 } catch (TException | BgpRouterException e) {
 852                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 853                 }
 854             }
 855         }
 856
 857         @Override
 858         protected EbgpMultihopReactor getDataTreeChangeListener() {
 859             return EbgpMultihopReactor.this;
 860         }
 861
 862         @Override
 863         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
 864             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(EbgpMultihop.class);
 865         }
 866
 867         @Override
 868         protected void remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 869             if (!isBGPEntityOwner()) {
 870                 return;
 871             }
 872             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 873             synchronized (BgpConfigurationManager.this) {
 874                 String peerIp = val.getPeerIp().getValue();
 875                 BgpRouter br = getClient(YANG_OBJ);
 876                 if (br == null) {
 877                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 878                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 879                     return;
 880                 }
 881                 try {
 882                     br.delEbgpMultihop(peerIp);
 883                 } catch (TException | BgpRouterException e) {
 884                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 885                 }
 886             }
 887         }
 888
 889         @Override
 890         protected void update(InstanceIdentifier<EbgpMultihop> iid,
 891                 EbgpMultihop oldval, EbgpMultihop newval) {
 892             if (!isBGPEntityOwner()) {
 893                 return;
 894             }
 895             LOG.error(YANG_OBJ + UPD_WARN);
 896         }
 897     }
 898
 899     public class UpdateSourceReactor
 900             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
 901             implements ClusteredDataTreeChangeListener<UpdateSource> {
 902
 903         private static final String YANG_OBJ = "update-source ";
 904
 905         public UpdateSourceReactor() {
 906             super(UpdateSource.class, UpdateSourceReactor.class);
 907         }
 908
 909         @Override
 910         protected void add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 911             if (!isBGPEntityOwner()) {
 912                 return;
 913             }
 914             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
 915             synchronized (BgpConfigurationManager.this) {
 916                 String peerIp = val.getPeerIp().getValue();
 917                 BgpRouter br = getClient(YANG_OBJ);
 918                 if (br == null) {
 919                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
 920                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 921                     return;
 922                 }
 923                 try {
 924                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
 925                 } catch (TException | BgpRouterException e) {
 926                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 927                 }
 928             }
 929         }
 930
 931         @Override
 932         protected UpdateSourceReactor getDataTreeChangeListener() {
 933             return UpdateSourceReactor.this;
 934         }
 935
 936         @Override
 937         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
 938             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(UpdateSource.class);
 939         }
 940
 941         @Override
 942         protected void remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 943             if (!isBGPEntityOwner()) {
 944                 return;
 945             }
 946             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
 947             synchronized (BgpConfigurationManager.this) {
 948                 String peerIp = val.getPeerIp().getValue();
 949                 BgpRouter br = getClient(YANG_OBJ);
 950                 if (br == null) {
 951                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 952                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 953                     return;
 954                 }
 955                 try {
 956                     br.delUpdateSource(peerIp);
 957                 } catch (TException | BgpRouterException e) {
 958                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 959                 }
 960             }
 961         }
 962
 963         @Override
 964         protected void update(InstanceIdentifier<UpdateSource> iid,
 965                 UpdateSource oldval, UpdateSource newval) {
 966             if (!isBGPEntityOwner()) {
 967                 return;
 968             }
 969             LOG.error(YANG_OBJ + UPD_WARN);
 970         }
 971     }
 972
 973     public class AddressFamiliesReactor
 974             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
 975             implements ClusteredDataTreeChangeListener<AddressFamilies> {
 976
 977         private static final String YANG_OBJ = "address-families ";
 978
 979         public AddressFamiliesReactor() {
 980             super(AddressFamilies.class, AddressFamiliesReactor.class);
 981         }
 982
 983         @Override
 984         protected void add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
 985             if (!isBGPEntityOwner()) {
 986                 return;
 987             }
 988             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
 989             synchronized (BgpConfigurationManager.this) {
 990                 String peerIp = val.getPeerIp().getValue();
 991                 BgpRouter br = getClient(YANG_OBJ);
 992                 if (br == null) {
 993                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
 994                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 995                     return;
 996                 }
 997                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
 998                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
 999                 try {
1000                     br.addAddressFamily(peerIp, afi, safi);
1001                 } catch (TException | BgpRouterException e) {
1002                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1003                 }
1004             }
1005         }
1006
1007         @Override
1008         protected AddressFamiliesReactor getDataTreeChangeListener() {
1009             return AddressFamiliesReactor.this;
1010         }
1011
1012         @Override
1013         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
1014             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(AddressFamilies.class);
1015         }
1016
1017         @Override
1018         protected void remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1019             if (!isBGPEntityOwner()) {
1020                 return;
1021             }
1022             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
1023             synchronized (BgpConfigurationManager.this) {
1024                 String peerIp = val.getPeerIp().getValue();
1025                 BgpRouter br = getClient(YANG_OBJ);
1026                 if (br == null) {
1027                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1028                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1029                     return;
1030                 }
1031                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1032                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1033                 try {
1034                     br.delAddressFamily(peerIp, afi, safi);
1035                 } catch (TException | BgpRouterException e) {
1036                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1037                 }
1038             }
1039         }
1040
1041         @Override
1042         protected void update(InstanceIdentifier<AddressFamilies> iid,
1043                 AddressFamilies oldval, AddressFamilies newval) {
1044             if (!isBGPEntityOwner()) {
1045                 return;
1046             }
1047             LOG.error(YANG_OBJ + UPD_WARN);
1048         }
1049     }
1050
1051     public class NetworksReactor
1052             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
1053             implements ClusteredDataTreeChangeListener<Networks> {
1054
1055         private static final String YANG_OBJ = "networks ";
1056
1057         public NetworksReactor() {
1058             super(Networks.class, NetworksReactor.class);
1059         }
1060
1061         @Override
1062         public NetworksReactor getDataTreeChangeListener() {
1063             return NetworksReactor.this;
1064         }
1065
1066         @Override
1067         protected void add(InstanceIdentifier<Networks> iid, Networks val) {
1068             if (!isBGPEntityOwner()) {
1069                 return;
1070             }
1071             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1072             synchronized (BgpConfigurationManager.this) {
1073                 String rd = val.getRd();
1074                 String pfxlen = val.getPrefixLen();
1075                 String nh = val.getNexthop().getValue();
1076                 BgpRouter br = getClient(YANG_OBJ);
1077                 if (br == null) {
1078                     LOG.debug("{} Unable to process add for rd {} prefix {} nexthop {}; {} {}", YANG_OBJ, rd, pfxlen,
1079                             nh, BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1080                     return;
1081                 }
1082                 Long label = val.getLabel();
1083                 int lbl = label == null ? qbgpConstants.LBL_NO_LABEL
1084                         : label.intValue();
1085                 int l3vni = val.getL3vni() == null ? qbgpConstants.LBL_NO_LABEL
1086                         : val.getL3vni().intValue();
1087                 int l2vni = val.getL2vni() == null ? qbgpConstants.LBL_NO_LABEL
1088                         : val.getL2vni().intValue();
1089
1090                 BgpControlPlaneType protocolType = val.getBgpControlPlaneType();
1091                 int ethernetTag = val.getEthtag().intValue();
1092                 String esi = val.getEsi();
1093                 String macaddress = val.getMacaddress();
1094                 EncapType encapType = val.getEncapType();
1095                 String routerMac = val.getRoutermac();
1096
1097                 try {
1098                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni, BgpUtil.convertToThriftProtocolType(protocolType),
1099                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1100                 } catch (TException | BgpRouterException e) {
1101                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1102                 }
1103             }
1104         }
1105
1106         @Override
1107         protected InstanceIdentifier<Networks> getWildCardPath() {
1108             return InstanceIdentifier.create(Bgp.class).child(Networks.class);
1109         }
1110
1111         @Override
1112         protected void remove(InstanceIdentifier<Networks> iid, Networks val) {
1113             if (!isBGPEntityOwner()) {
1114                 return;
1115             }
1116             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1117             synchronized (BgpConfigurationManager.this) {
1118                 String rd = val.getRd();
1119                 String pfxlen = val.getPrefixLen();
1120                 BgpRouter br = getClient(YANG_OBJ);
1121                 if (br == null) {
1122                     LOG.debug("{} Unable to process remove for rd {} prefix {}; {} {}", YANG_OBJ, rd, pfxlen,
1123                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1124                     return;
1125                 }
1126                 Long label = val.getLabel();
1127                 int lbl = label == null ? 0 : label.intValue();
1128                 if (rd == null && lbl > 0) {
1129                     //LU prefix is being deleted.
1130                     rd = Integer.toString(lbl);
1131                 }
1132                 try {
1133                     br.delPrefix(rd, pfxlen);
1134                 } catch (TException | BgpRouterException e) {
1135                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1136                 }
1137             }
1138         }
1139
1140         /**get the value AFI from a prefix as "x.x.x.x/x".
1141          *
1142          * @param pfxlen the prefix to get an afi
1143          * @return the afi value as you are need
1144          */
1145         public  int  testValueAFI(String pfxlen) {
1146             int afiNew = af_afi.AFI_IP.getValue();
1147             try {
1148                 String ipOnly = pfxlen.substring(0, pfxlen.lastIndexOf("/"));
1149                 java.net.Inet6Address.getByName(ipOnly);
1150                 afiNew = af_afi.AFI_IPV6.getValue();
1151             } catch (java.net.UnknownHostException e) {
1152                 //ce n'est pas de l'ipv6
1153             }
1154             return afiNew;
1155         }
1156
1157
1158         @Override
1159         protected void update(final InstanceIdentifier<Networks> iid,
1160                 final Networks oldval, final Networks newval) {
1161             if (!isBGPEntityOwner()) {
1162                 return;
1163             }
1164             if (oldval.equals(newval)) {
1165                 //Update: OLD and New values are same, no need to trigger remove/add.
1166                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1167                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1168                 return;
1169             }
1170             LOG.debug("update networks old val RD: {}, Prefix: {}, Label: {}, NH: {} "
1171                             + "new val RD: {}, Prefix: {}, Label: {}, NH: {}",
1172                     oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop(),
1173                     newval.getRd(), newval.getPrefixLen(), newval.getLabel(), newval.getNexthop());
1174             remove(iid, oldval);
1175             add(iid, newval);
1176         }
1177     }
1178
1179     static Timer timer = new Timer();
1180
1181     public class VrfsReactor
1182             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1183             implements ClusteredDataTreeChangeListener<Vrfs> {
1184
1185         private static final String YANG_OBJ = "vrfs ";
1186
1187         public VrfsReactor() {
1188             super(Vrfs.class, VrfsReactor.class);
1189         }
1190
1191         @Override
1192         protected void add(InstanceIdentifier<Vrfs> iid, Vrfs vrfs) {
1193             if (!isBGPEntityOwner()) {
1194                 return;
1195             }
1196             LOG.debug("received add Vrfs config value {}", vrfs.getRd());
1197             synchronized (BgpConfigurationManager.this) {
1198                 String rd = vrfs.getRd();
1199                 BgpRouter br = getClient(YANG_OBJ);
1200                 if (br == null) {
1201                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1202                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1203                     return;
1204                 }
1205                 try {
1206                     List<AddressFamiliesVrf> vrfAddrFamilyList = vrfs.getAddressFamiliesVrf();
1207                     for (AddressFamiliesVrf vrfAddrFamily : vrfAddrFamilyList) {
1208                         /*add to br the new vrfs arguments*/
1209                         br.addVrf(BgpUtil.getLayerType(vrfAddrFamily), rd, vrfs.getImportRts(), vrfs.getExportRts()
1210                         );
1211                     }
1212                     /*add to br the vrfs contained in mapNewAdFamily*/
1213                     List<AddressFamiliesVrf> vrfAddrFamilyListFromMap = mapNewAdFamily.get(rd);
1214                     if (vrfAddrFamilyListFromMap == null) {
1215                         return;
1216                     }
1217
1218                     for (AddressFamiliesVrf adf : vrfAddrFamilyListFromMap) {
1219                         if (vrfAddrFamilyList.contains(adf)) {
1220                             mapNewAdFamily.remove(rd);
1221                         } else  if (adf != null) {
1222
1223                             br.addVrf(BgpUtil.getLayerType(adf), rd, vrfs.getImportRts(), vrfs.getExportRts()
1224                             );
1225                             // remove AddressFamiliesVrf which was already added to BGP
1226                             vrfAddrFamilyListFromMap.remove(adf);
1227                             if (vrfAddrFamilyListFromMap.isEmpty()) {
1228                                 // remove Vrf entry from temp mapNewAdFamily if all its AddressFamiliesVrf was
1229                                 // added to BGP
1230                                 mapNewAdFamily.remove(rd);
1231                             }
1232                         }
1233                     }
1234                 } catch (TException | BgpRouterException e) {
1235                     LOG.error("{} get {}, Add received exception", YANG_OBJ, ADD_WARN, e);
1236                 }
1237             }
1238         }
1239
1240         @Override
1241         protected VrfsReactor getDataTreeChangeListener() {
1242             return VrfsReactor.this;
1243         }
1244
1245         @Override
1246         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1247             return InstanceIdentifier.create(Bgp.class).child(Vrfs.class);
1248         }
1249
1250         @Override
1251         protected void remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1252             if (!isBGPEntityOwner()) {
1253                 return;
1254             }
1255             LOG.debug("received remove Vrfs config val {}", val.getRd());
1256             synchronized (BgpConfigurationManager.this) {
1257                 String rd = val.getRd();
1258                 BgpRouter br = getClient(YANG_OBJ);
1259                 if (br == null) {
1260                     LOG.debug("{} Unable to process remove for rd {}; {} {}", YANG_OBJ, rd,
1261                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1262                     return;
1263                 }
1264                 try {
1265                     List<AddressFamiliesVrf> adf = mapNewAdFamily.get(rd);
1266                     adf = adf != null ? adf : new ArrayList<>();
1267                     for (AddressFamiliesVrf s : val.getAddressFamiliesVrf()) {
1268                         br.delVrf(rd, s.getAfi(), s.getSafi());
1269                         adf.remove(s);// remove in the map the vrf in waiting for advertise quagga
1270                     }
1271                     if (adf.isEmpty()) {
1272                         mapNewAdFamily.remove(rd);
1273                     }
1274                 } catch (TException | BgpRouterException e) {
1275                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1276                 }
1277             }
1278         }
1279
1280         @Override
1281         protected void update(InstanceIdentifier<Vrfs> iid,
1282                 Vrfs oldval, Vrfs newval) {
1283             if (oldval != null && newval != null) {
1284                 LOG.debug("received update Vrfs config val {}, VRFS: Update getting triggered for VRFS rd {}",
1285                         newval.getRd(), oldval.getRd());
1286             } else {
1287                 LOG.debug("received update Vrfs config val {}, from old vrf {}",
1288                         newval, oldval);
1289             }
1290             if (!isBGPEntityOwner()) {
1291                 return;
1292             }
1293
1294             List<AddressFamiliesVrf> adFamilyVrfToDel = new ArrayList<>();
1295             List<AddressFamiliesVrf> adFamilyVrfToAdd = new ArrayList<>();
1296             List<AddressFamiliesVrf> oldlistAdFamilies = new ArrayList<>();
1297             List<AddressFamiliesVrf> newlistAdFamilies = new ArrayList<>();
1298             if (oldval != null) {
1299                 oldlistAdFamilies = oldval.getAddressFamiliesVrf() == null
1300                         ? new ArrayList<>() : oldval.getAddressFamiliesVrf();
1301             }
1302             if (newval != null) {
1303                 newlistAdFamilies = newval.getAddressFamiliesVrf() == null
1304                         ? new ArrayList<>() : newval.getAddressFamiliesVrf();
1305             }
1306             /*find old AddressFamily to remove from new configuration*/
1307             for (AddressFamiliesVrf adVrf : oldlistAdFamilies) {
1308                 if (!newlistAdFamilies.contains(adVrf)) {
1309                     adFamilyVrfToDel.add(adVrf);
1310                 }
1311             }
1312             /*find new AddressFamily to add to unexisting configuration*/
1313             for (AddressFamiliesVrf adVrf : newlistAdFamilies) {
1314                 if (!oldlistAdFamilies.contains(adVrf)) {
1315                     adFamilyVrfToAdd.add(adVrf);
1316                 }
1317             }
1318             String rd = newval != null ? newval.getRd() : null;
1319             if (rd != null) {
1320                 BgpRouter br = getClient(YANG_OBJ);
1321                 if (br == null) {
1322                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1323                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1324                     return;
1325                 }
1326
1327                 for (AddressFamiliesVrf adfvrf : adFamilyVrfToAdd) {
1328                     try {
1329                         LOG.debug("call addVRf rd {} afi {} safi {}", rd, adfvrf.getAfi(), adfvrf.getSafi());
1330                         br.addVrf(BgpUtil.getLayerType(adfvrf), rd, newval.getImportRts(),
1331                                 newval.getExportRts());
1332                     } catch (TException | BgpRouterException e) {
1333                         LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1334                     }
1335                 }
1336
1337                 for (AddressFamiliesVrf adfToDel : adFamilyVrfToDel) {
1338                     try {
1339                         LOG.debug("call delVRf rd {} afi {} safi {}", rd, adfToDel.getAfi(), adfToDel.getSafi());
1340                         br.delVrf(rd, adfToDel.getAfi(), adfToDel.getSafi());
1341                     } catch (TException | BgpRouterException e) {
1342                         LOG.error("{} delVrf received exception; {}", YANG_OBJ, ADD_WARN, e);
1343                     }
1344                 }
1345             }
1346         }
1347     }
1348
1349     protected void activateMIP() {
1350         try {
1351             LOG.trace("BgpReactor: Executing MIP Activate command");
1352             Runtime.getRuntime().exec("cluster ip -a sdnc_bgp_mip");
1353             Runtime.getRuntime().exec("cluster ip -a sdnc_os_mip");
1354             LOG.trace("bgpMIP Activated");
1355
1356         } catch (IOException io) {
1357             LOG.error("IO Exception got while activating mip: {}", io.getMessage());
1358         }
1359     }
1360
1361     public class BgpReactor
1362             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1363             implements ClusteredDataTreeChangeListener<Bgp> {
1364
1365         private static final String YANG_OBJ = "Bgp ";
1366
1367         public BgpReactor() {
1368             super(Bgp.class, BgpReactor.class);
1369         }
1370
1371
1372         @Override
1373         protected void add(InstanceIdentifier<Bgp> iid, Bgp val) {
1374             LOG.debug("received add Bgp config");
1375
1376             try {
1377                 initer.await();
1378             } catch (InterruptedException e) {
1379                 // Ignored
1380             }
1381             synchronized (BgpConfigurationManager.this) {
1382                 config = val;
1383                 if (!isBGPEntityOwner()) {
1384                     return;
1385                 }
1386                 activateMIP();
1387             }
1388         }
1389
1390         @Override
1391         protected BgpReactor getDataTreeChangeListener() {
1392             return BgpReactor.this;
1393         }
1394
1395         @Override
1396         protected InstanceIdentifier<Bgp> getWildCardPath() {
1397             return InstanceIdentifier.create(Bgp.class);
1398         }
1399
1400         @Override
1401         protected void remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1402             if (!isBGPEntityOwner()) {
1403                 return;
1404             }
1405             LOG.debug("received remove Bgp config");
1406
1407             config = null;
1408         }
1409
1410         @Override
1411         protected void update(InstanceIdentifier<Bgp> iid,
1412                 Bgp oldval, Bgp newval) {
1413             if (!isBGPEntityOwner()) {
1414                 return;
1415             }
1416
1417             config = newval;
1418         }
1419     }
1420
1421     @SuppressWarnings("deprecation")
1422     public class MultipathReactor
1423             extends AsyncDataTreeChangeListenerBase<Multipath, MultipathReactor>
1424             implements ClusteredDataTreeChangeListener<Multipath> {
1425
1426         private static final String YANG_OBJ = "multipath ";
1427
1428         public MultipathReactor() {
1429             super(Multipath.class, MultipathReactor.class);
1430         }
1431
1432
1433         @Override
1434         protected MultipathReactor getDataTreeChangeListener() {
1435             return MultipathReactor.this;
1436         }
1437
1438         @Override
1439         protected InstanceIdentifier<Multipath> getWildCardPath() {
1440             return InstanceIdentifier.create(Bgp.class).child(Multipath.class);
1441         }
1442
1443         @Override
1444         protected void remove(InstanceIdentifier<Multipath> iid, Multipath val) {
1445             executor.execute(new MultipathStatusChange(val));
1446         }
1447
1448         @Override
1449         protected void update(InstanceIdentifier<Multipath> iid, Multipath oldval, Multipath newval) {
1450             executor.execute(new MultipathStatusChange(newval));
1451         }
1452
1453         @Override
1454         protected void add(InstanceIdentifier<Multipath> key, Multipath dataObjectModification) {
1455             executor.execute(new MultipathStatusChange(dataObjectModification));
1456         }
1457
1458         class MultipathStatusChange implements Runnable {
1459
1460             Multipath multipath;
1461
1462             MultipathStatusChange(Multipath multipath) {
1463                 this.multipath = multipath;
1464             }
1465
1466             @Override
1467             public void run() {
1468                 if (isBGPEntityOwner()) {
1469                     synchronized (BgpConfigurationManager.this) {
1470
1471                         BgpRouter br = getClient(YANG_OBJ);
1472
1473                         if (br != null) {
1474                             af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1475                             af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1476
1477                             try {
1478                                 if (multipath.isMultipathEnabled()) {
1479                                     br.enableMultipath(afi, safi);
1480                                 } else {
1481                                     br.disableMultipath(afi, safi);
1482                                 }
1483                             } catch (TException | BgpRouterException e) {
1484                                 LOG.error("{} received exception", YANG_OBJ, e);
1485                             }
1486                         }
1487                     }
1488                 }
1489             }
1490
1491         }
1492
1493         @Override
1494         public void close() {
1495             super.close();
1496         }
1497     }
1498
1499     @SuppressWarnings("deprecation")
1500     public class VrfMaxpathReactor
1501             extends AsyncDataTreeChangeListenerBase<VrfMaxpath, VrfMaxpathReactor>
1502             implements ClusteredDataTreeChangeListener<VrfMaxpath> {
1503
1504         private static final String YANG_OBJ = "vrfMaxpath ";
1505
1506         public VrfMaxpathReactor() {
1507             super(VrfMaxpath.class, VrfMaxpathReactor.class);
1508         }
1509
1510
1511         @Override
1512         protected VrfMaxpathReactor getDataTreeChangeListener() {
1513             return VrfMaxpathReactor.this;
1514         }
1515
1516         @Override
1517         protected InstanceIdentifier<VrfMaxpath> getWildCardPath() {
1518             return InstanceIdentifier.create(Bgp.class).child(VrfMaxpath.class);
1519         }
1520
1521         class VrfMaxPathConfigurator implements Runnable {
1522
1523             VrfMaxpath vrfMaxpathVal;
1524
1525             VrfMaxPathConfigurator(VrfMaxpath vrfMaxPathVal) {
1526                 this.vrfMaxpathVal = vrfMaxPathVal;
1527             }
1528
1529             @Override
1530             public void run() {
1531                 if (isBGPEntityOwner()) {
1532                     synchronized (BgpConfigurationManager.this) {
1533                         BgpRouter br = getClient(YANG_OBJ);
1534                         if (br != null) {
1535                             try {
1536                                 br.multipaths(vrfMaxpathVal.getRd(), vrfMaxpathVal.getMaxpaths());
1537                                 LOG.debug("Maxpath for vrf {} is {}", vrfMaxpathVal.getRd(),
1538                                         vrfMaxpathVal.getMaxpaths());
1539                             } catch (TException | BgpRouterException e) {
1540                                 LOG.error("{} received exception", YANG_OBJ, e);
1541                             }
1542                         }
1543                     }
1544                 }
1545             }
1546         }
1547
1548         @Override
1549         protected void remove(InstanceIdentifier<VrfMaxpath> iid, VrfMaxpath vrfMaxPathVal) {
1550             executor.execute(new VrfMaxPathConfigurator(vrfMaxPathVal));
1551         }
1552
1553         @Override
1554         protected void update(InstanceIdentifier<VrfMaxpath> iid,
1555                               VrfMaxpath oldval, VrfMaxpath newval) {
1556             if (!Objects.equals(oldval.getMaxpaths(), newval.getMaxpaths())) {
1557                 executor.execute(new VrfMaxPathConfigurator(newval));
1558             }
1559         }
1560
1561         @Override
1562         protected void add(InstanceIdentifier<VrfMaxpath> instanceIdentifier, VrfMaxpath vrfMaxpathVal) {
1563             executor.execute(new VrfMaxPathConfigurator(vrfMaxpathVal));
1564         }
1565
1566         @Override
1567         public void close() {
1568             super.close();
1569         }
1570     }
1571
1572     public boolean isIpAvailable(String odlip) {
1573
1574         try {
1575             if (odlip != null) {
1576                 if ("127.0.0.1".equals(odlip)) {
1577                     return true;
1578                 }
1579                 Enumeration<NetworkInterface> networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces();
1580                 while (networkInterfaceEnumeration.hasMoreElements()) {
1581                     NetworkInterface networkInterface = networkInterfaceEnumeration.nextElement();
1582                     Enumeration<InetAddress>  inetAddressEnumeration = networkInterface.getInetAddresses();
1583                     while (inetAddressEnumeration.hasMoreElements()) {
1584                         InetAddress inetAddress = inetAddressEnumeration.nextElement();
1585                         if (odlip.equals(inetAddress.getHostAddress())) {
1586                             return true;
1587                         }
1588                     }
1589                 }
1590             }
1591         } catch (SocketException e) {
1592             // Ignored?
1593         }
1594         return false;
1595     }
1596
1597     public long getStalePathtime(int defValue, AsId asId) {
1598         long spt = 0;
1599         try {
1600             spt = getConfig().getGracefulRestart().getStalepathTime();
1601         } catch (NullPointerException e) {
1602             try {
1603                 spt = asId.getStalepathTime();
1604                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1605             } catch (NullPointerException ignore) {
1606                 LOG.trace("BGP AS id is not set using graceful");
1607                 spt = defValue;
1608             }
1609         }
1610         if (spt == 0) {
1611             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1612             spt = defValue;
1613         }
1614         return spt;
1615     }
1616
1617     public static boolean isValidConfigBgpHostPort(String bgpHost, int bgpPort) {
1618         if (!bgpHost.equals(DEF_CHOST) && bgpPort != Integer.parseInt(DEF_CPORT)) {
1619             return true;
1620         } else {
1621             return false;
1622         }
1623     }
1624
1625     public synchronized void bgpRestarted() {
1626         /*
1627          * If there a thread which in the process of stale cleanup, cancel it
1628          * and start a new thread (to avoid processing same again).
1629          */
1630         if (previousReplayJobInProgress()) {
1631             cancelPreviousReplayJob();
1632         }
1633         Runnable task = () -> {
1634             try {
1635                 LOG.info("running bgp replay task ");
1636                 if (get() == null) {
1637                     String host = getConfigHost();
1638                     int port = getConfigPort();
1639                     LOG.info("connecting  to bgp host {} ", host);
1640                     bgpRouter.connect(host, port);
1641                     LOG.info("no config to push in bgp replay task ");
1642                     return;
1643                 }
1644                 setStaleStartTime(System.currentTimeMillis());
1645                 LOG.info("started creating stale fibDSWriter  map ");
1646                 createStaleFibMap();
1647                 setStaleEndTime(System.currentTimeMillis());
1648                 LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
1649                 LOG.info("started bgp config replay ");
1650                 setCfgReplayStartTime(System.currentTimeMillis());
1651                 boolean replaySucceded = replay();
1652                 setCfgReplayEndTime(System.currentTimeMillis());
1653                 LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1654                 if (replaySucceded) {
1655                     LOG.info("starting the stale cleanup timer");
1656                     long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1657                     setStaleCleanupTime(routeSyncTime);
1658                     routeCleanupFuture = executor.schedule(new RouteCleanup(), routeSyncTime, TimeUnit.SECONDS);
1659                 } else {
1660                     staledFibEntriesMap.clear();
1661                 }
1662             } catch (InterruptedException | TimeoutException | ExecutionException eCancel) {
1663                 LOG.error("Stale Cleanup Task Cancelled", eCancel);
1664             }
1665         };
1666         lastReplayJobFt = executor.submit(task);
1667     }
1668
1669     private boolean previousReplayJobInProgress() {
1670         return lastReplayJobFt != null && !lastReplayJobFt.isDone();
1671     }
1672
1673     private void cancelPreviousReplayJob() {
1674         try {
1675             LOG.error("cancelling already running bgp replay task");
1676             if (lastReplayJobFt != null) {
1677                 lastReplayJobFt.cancel(true);
1678                 lastReplayJobFt = null;
1679                 staledFibEntriesMap.clear();
1680             }
1681             if (routeCleanupFuture != null) {
1682                 routeCleanupFuture.cancel(true);
1683                 routeCleanupFuture = null;
1684                 staledFibEntriesMap.clear();
1685             }
1686             Thread.sleep(2000);
1687         } catch (InterruptedException e) {
1688             LOG.error("Failed to cancel previous replay job ", e);
1689         }
1690     }
1691
1692     private void doRouteSync() {
1693         LOG.error("Starting BGP route sync");
1694         try {
1695             bgpRouter.initRibSync(bgpSyncHandle);
1696         } catch (BgpRouterException e) {
1697             LOG.error("Route sync aborted, exception when initializing", e);
1698             return;
1699         }
1700         while (bgpSyncHandle.getState() != BgpSyncHandle.DONE) {
1701             for (af_afi afi : af_afi.values()) {
1702                 Routes routes = null;
1703                 try {
1704                     routes = bgpRouter.doRibSync(bgpSyncHandle, afi);
1705                 } catch (TException | BgpRouterException e) {
1706                     LOG.error("Route sync aborted, exception when syncing", e);
1707                     return;
1708                 }
1709                 Iterator<Update> updates = routes.getUpdatesIterator();
1710                 while (updates.hasNext()) {
1711                     Update update = updates.next();
1712                     String rd = update.getRd();
1713                     String nexthop = update.getNexthop();
1714
1715                     // TODO: decide correct label here
1716                     int label = update.getL3label();
1717                     int l2label = update.getL2label();
1718
1719                     String prefix = update.getPrefix();
1720                     int plen = update.getPrefixlen();
1721
1722
1723                     // TODO: protocol type will not be available in "update"
1724                     // use "rd" to query vrf table and obtain the protocol_type.
1725                     // Currently using PROTOCOL_EVPN as default.
1726                     onUpdatePushRoute(
1727                            protocol_type.PROTOCOL_EVPN,
1728                            rd,
1729                            prefix,
1730                            plen,
1731                            nexthop,
1732                            update.getMacaddress(),
1733                            label,
1734                            l2label,
1735                            update.getRoutermac(),
1736                            afi);
1737                 }
1738             }
1739         }
1740         try {
1741             LOG.error("Ending BGP route-sync");
1742             bgpRouter.endRibSync(bgpSyncHandle);
1743         } catch (BgpRouterException e) {
1744             // Ignored?
1745         }
1746     }
1747
1748     public void addTepToElanDS(String rd, String tepIp, String mac, Long l2vni) {
1749         boolean needUpdate = addToRt2TepMap(rd, tepIp, mac, l2vni);
1750         if (needUpdate) {
1751             LOG.info("Adding tepIp {} with RD {} to ELan DS", tepIp, rd);
1752             bgpUtil.addTepToElanInstance(rd, tepIp);
1753         } else {
1754             LOG.debug("Skipping the Elan update for RT2 from tep {} rd {}", tepIp, rd);
1755         }
1756     }
1757
1758     public void deleteTepfromElanDS(String rd, String tepIp, String mac) {
1759         boolean needUpdate = deleteFromRt2TepMap(rd, tepIp, mac);
1760         if (needUpdate) {
1761             LOG.info("Deleting tepIp {} with RD {} to ELan DS", tepIp, rd);
1762             bgpUtil.deleteTepFromElanInstance(rd, tepIp);
1763         } else {
1764             LOG.debug("Skipping the Elan update for RT2 withdraw from tep {} rd {}", tepIp, rd);
1765         }
1766     }
1767
1768     /* onUpdatePushRoute
1769      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1770      *  - Entry compare shall include NextHop, Label.
1771      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1772      *  - If entry not found, add to FIB Config DS.
1773      *  - If entry found, but either Label/NextHop doesn't match.
1774      *      - Update FIB Config DS with modified values.
1775      *      - delete from Stale Map.
1776      */
1777
1778     public void onUpdatePushRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
1779                                   String macaddress, int label, int l2label, String routermac, af_afi afi) {
1780         boolean addroute = false;
1781         boolean macupdate = false;
1782         long l3vni = 0L;
1783         VrfEntry.EncapType encapType = VrfEntry.EncapType.Mplsgre;
1784         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1785             encapType = VrfEntry.EncapType.Vxlan;
1786             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
1787             if (vpnInstanceOpDataEntry != null) {
1788                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
1789                     LOG.info("Got RT2 route for RD {} l3label {} l2label {} from tep {} with mac {} remote RD {}",
1790                             vpnInstanceOpDataEntry.getVpnInstanceName(), label, l2label, nextHop, macaddress, rd);
1791                     addTepToElanDS(rd, nextHop, macaddress, (long)l2label);
1792                     macupdate = true;
1793                 } else {
1794                     l3vni = vpnInstanceOpDataEntry.getL3vni();
1795                 }
1796             } else {
1797                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1798                 return;
1799             }
1800         }
1801
1802         if (!staledFibEntriesMap.isEmpty()) {
1803             // restart Scenario, as MAP is not empty.
1804             Map<String, Long> map = staledFibEntriesMap.get(rd);
1805             if (map != null) {
1806                 String prefixNextHop = appendNextHopToPrefix(prefix + "/" + plen, nextHop);
1807                 Long labelInStaleMap = map.get(prefixNextHop);
1808                 if (null == labelInStaleMap) {
1809                     // New Entry, which happened to be added during restart.
1810                     addroute = true;
1811                 } else {
1812                     map.remove(prefixNextHop);
1813                     if (isRouteModified(label, labelInStaleMap)) {
1814                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1815                         // Existing entry, where in Label got modified during restart
1816                         addroute = true;
1817                     }
1818                 }
1819             } else {
1820                 LOG.debug("rd {} map is null while processing prefix {} ", rd, prefix);
1821                 addroute = true;
1822             }
1823         } else {
1824             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1825             addroute = true;
1826         }
1827         if (macupdate) {
1828             LOG.info("ADD: Adding Mac Fib entry rd {} mac{} nexthop {} l2vni {}", rd, macaddress, nextHop, l2label);
1829             fibDSWriter.addMacEntryToDS(rd, macaddress, prefix, Collections.singletonList(nextHop),
1830                     encapType, l2label, routermac, RouteOrigin.BGP);
1831             LOG.info("ADD: Added Mac Fib entry rd {} prefix {} nexthop {} label {}", rd, macaddress, nextHop, l2label);
1832         } else if (addroute) {
1833             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {} afi {}",
1834                     rd, prefix, nextHop, label, afi);
1835             // TODO: modify addFibEntryToDS signature
1836             List<String> nextHopList = Collections.singletonList(nextHop);
1837             fibDSWriter.addFibEntryToDS(rd, prefix + "/" + plen, nextHopList, encapType, label, l3vni,
1838                                         routermac, RouteOrigin.BGP);
1839             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1840             String vpnName = bgpUtil.getVpnNameFromRd(rd);
1841             if (vpnName != null) {
1842                 vpnLinkService.leakRouteIfNeeded(vpnName, prefix, nextHopList, label, RouteOrigin.BGP,
1843                                                  NwConstants.ADD_FLOW);
1844             }
1845         }
1846     }
1847
1848     public void onUpdateWithdrawRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
1849             String macaddress) {
1850         long vni = 0L;
1851         boolean macupdate = false;
1852         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1853             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
1854             if (vpnInstanceOpDataEntry != null) {
1855                 vni = vpnInstanceOpDataEntry.getL3vni();
1856                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
1857                     LOG.debug("Got RT2 withdraw for RD {} {} from tep {} with mac {} remote RD {}",
1858                             vpnInstanceOpDataEntry.getVpnInstanceName(), vni, nextHop, macaddress, rd);
1859                     deleteTepfromElanDS(rd, nextHop, macaddress);
1860                     LOG.debug("For rd {}. skipping fib update", rd);
1861                     macupdate = true;
1862                 }
1863             } else {
1864                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1865                 return;
1866             }
1867         }
1868         if (macupdate) {
1869             LOG.info("Removing Mac Fib entry rd {} mac{} nexthop {} ", rd, macaddress, nextHop);
1870             fibDSWriter.removeMacEntryFromDS(rd, macaddress);
1871             LOG.info("Removed Mac Fib entry rd {} prefix {} nexthop {} ", rd, macaddress, nextHop);
1872         } else {
1873             LOG.info("REMOVE: Removing Fib entry rd {} prefix {}", rd, prefix);
1874             fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nextHop);
1875             LOG.info("REMOVE: Removed Fib entry rd {} prefix {}", rd, prefix);
1876         }
1877     }
1878
1879     //TODO: below function is for testing purpose with cli
1880     public void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
1881         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
1882         fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nexthop);
1883         String vpnName = bgpUtil.getVpnNameFromRd(rd);
1884         if (vpnName != null) {
1885             vpnLinkService.leakRouteIfNeeded(vpnName, prefix, null /*nextHopList*/, 0 /*INVALID_LABEL*/,
1886                                              RouteOrigin.BGP, NwConstants.DEL_FLOW);
1887         }
1888     }
1889
1890     private static boolean isRouteModified(int label, Long labelInStaleMap) {
1891         return labelInStaleMap != null && !labelInStaleMap.equals(Long.valueOf(label));
1892     }
1893
1894     static class ReplayNbr {
1895         Neighbors nbr;
1896         boolean shouldRetry = false;
1897
1898         public Neighbors getNbr() {
1899             return nbr;
1900         }
1901
1902         public boolean isShouldRetry() {
1903             return shouldRetry;
1904         }
1905
1906         public void setShouldRetry(boolean retryNbr) {
1907             this.shouldRetry = retryNbr;
1908         }
1909
1910         ReplayNbr(Neighbors nbr, boolean shouldRetry) {
1911             this.nbr = nbr;
1912             this.shouldRetry = shouldRetry;
1913         }
1914     }
1915
1916     private static boolean replayNbrConfig(List<Neighbors> neighbors, BgpRouter br) {
1917         if (neighbors == null || neighbors.isEmpty()) {
1918             LOG.error("Replaying nbr configuration, received NULL list ");
1919             return true;
1920         }
1921
1922         List<ReplayNbr> replayNbrList = new ArrayList<>();
1923         for (Neighbors nbr : neighbors) {
1924             if (nbr != null) {
1925                 replayNbrList.add(new ReplayNbr(nbr, true));
1926             }
1927         }
1928         final int numberOfNbrRetries = 3;
1929         RetryOnException nbrRetry = new RetryOnException(numberOfNbrRetries);
1930         do {
1931             for (ReplayNbr replayNbr : replayNbrList) {
1932                 if (!replayNbr.isShouldRetry()) {
1933                     continue;
1934                 }
1935                 boolean replayDone = false;
1936                 LOG.debug("Replaying addNbr {}", replayNbr.getNbr().getAddress().getValue());
1937                 replayDone = false;
1938                 try {
1939                     final String md5password = extractMd5Secret(replayNbr.getNbr());
1940                     br.addNeighbor(replayNbr.getNbr().getAddress().getValue(),
1941                             replayNbr.getNbr().getRemoteAs().longValue(), md5password);
1942                     UpdateSource us = replayNbr.getNbr().getUpdateSource();
1943                     if (us != null) {
1944                         LOG.debug("Replaying updatesource along with nbr: {} US-ip: {} to peer {}",
1945                                 replayNbr.getNbr().getAddress().getValue(),
1946                                 us.getSourceIp().getValue(),
1947                                 us.getPeerIp().getValue());
1948                         br.addUpdateSource(us.getPeerIp().getValue(),
1949                                 us.getSourceIp().getValue());
1950                     }
1951                     replayDone = true;
1952                 } catch (TException | BgpRouterException eNbr) {
1953                     LOG.debug("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), eNbr);
1954                 }
1955                 boolean replaySuccess = true;
1956                 replaySuccess = replaySuccess && replayDone;
1957                 LOG.debug("Replay addNbr {} successful", replayNbr.getNbr().getAddress().getValue());
1958
1959                 //Update Source handling
1960                 UpdateSource us = replayNbr.getNbr().getUpdateSource();
1961                 if (replayDone == false && us != null) {
1962                     LOG.debug("Replaying updatesource {} to peer {}", us.getSourceIp().getValue(),
1963                             us.getPeerIp().getValue());
1964                     replayDone = false;
1965                     try {
1966                         br.addUpdateSource(us.getPeerIp().getValue(),
1967                                 us.getSourceIp().getValue());
1968                         replayDone = true;
1969                     } catch (TException | BgpRouterException eUs) {
1970                         LOG.debug("Replaying UpdateSource for Nbr {}, exception:",
1971                                 replayNbr.getNbr().getAddress().getValue(), eUs);
1972                     }
1973                     LOG.debug("Replay updatesource {} successful", us.getSourceIp().getValue());
1974                     replaySuccess = replaySuccess && replayDone;
1975                 }
1976                 //Ebgp Multihope
1977                 EbgpMultihop en = replayNbr.getNbr().getEbgpMultihop();
1978                 if (en != null) {
1979                     replayDone = false;
1980                     try {
1981                         br.addEbgpMultihop(en.getPeerIp().getValue(),
1982                                 en.getNhops().intValue());
1983                         replayDone = true;
1984                     } catch (TException | BgpRouterException eEbgpMhop) {
1985                         LOG.debug("Replaying EbgpMultihop for Nbr {}, exception: ",
1986                                 replayNbr.getNbr().getAddress().getValue(), eEbgpMhop);
1987                     }
1988                     replaySuccess = replaySuccess && replayDone;
1989                 }
1990
1991                 //afs
1992                 List<AddressFamilies> afs = replayNbr.getNbr().getAddressFamilies();
1993                 if (afs != null) {
1994                     for (AddressFamilies af : afs) {
1995                         af_afi afi = af_afi.findByValue(af.getAfi().intValue());
1996                         af_safi safi = af_safi.findByValue(af.getSafi().intValue());
1997                         replayDone = false;
1998                         try {
1999                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
2000                             replayDone = true;
2001                         } catch (TException | BgpRouterException eAFs) {
2002                             LOG.debug("Replaying AddressFamily for Nbr {}, exception:",
2003                                     replayNbr.getNbr().getAddress().getValue(), eAFs);
2004                         }
2005                         replaySuccess = replaySuccess && replayDone;
2006                     }
2007                 }
2008                 //replay is success --> no need to replay this nbr in next iteration.
2009                 replayNbr.setShouldRetry(replaySuccess ? false : true);
2010             }
2011         } while (nbrRetry.decrementAndRetry());
2012         boolean replaySuccess = true;
2013         for (ReplayNbr replayNbr : replayNbrList) {
2014             replaySuccess = replaySuccess && !replayNbr.isShouldRetry();
2015         }
2016         return replaySuccess;
2017     }
2018
2019     public String getConfigHost() {
2020         if (config == null) {
2021             return hostStartup;
2022         }
2023         ConfigServer ts = config.getConfigServer();
2024         return ts == null ? hostStartup : ts.getHost().getValue();
2025     }
2026
2027     public int getConfigPort() {
2028         if (config == null) {
2029             return Integer.parseInt(portStartup);
2030         }
2031         ConfigServer ts = config.getConfigServer();
2032         return ts == null ? Integer.parseInt(portStartup) :
2033                 ts.getPort().intValue();
2034     }
2035
2036     public Bgp getConfig() {
2037         AtomicInteger bgpDSretryCount = new AtomicInteger(DS_RETRY_COUNT);
2038         while (0 != bgpDSretryCount.decrementAndGet()) {
2039             try {
2040                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
2041                         InstanceIdentifier.create(Bgp.class)).orNull();
2042             } catch (ReadFailedException e) {
2043                 //Config DS may not be up, so sleep for 1 second and retry
2044                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
2045                 try {
2046                     Thread.sleep(WAIT_TIME_BETWEEN_EACH_TRY_MILLIS);
2047                 } catch (InterruptedException timerEx) {
2048                     LOG.debug("WAIT_TIME_BETWEEN_EACH_TRY_MILLIS, Timer got interrupted while waiting for"
2049                             + "config DS availability", timerEx);
2050                 }
2051             }
2052         }
2053         LOG.error("failed to get bgp config");
2054         return null;
2055     }
2056
2057     @SuppressWarnings("checkstyle:IllegalCatch")
2058     public synchronized boolean replay() throws InterruptedException, TimeoutException, ExecutionException {
2059         boolean replaySucceded = true;
2060         String host = getConfigHost();
2061         int port = getConfigPort();
2062         LOG.error("connecting  to bgp host {} ", host);
2063         boolean res = bgpRouter.connect(host, port);
2064         if (!res) {
2065             LOG.error("Cannot connect to BGP config server at {}:{}{}", host, port,
2066                     config != null ? "; Configuration Replay aborted" : "");
2067             return replaySucceded;
2068         }
2069         config = getConfig();
2070         if (config == null) {
2071             LOG.error("bgp config is empty nothing to push to bgp");
2072             return replaySucceded;
2073         }
2074         BgpRouter br = bgpRouter;
2075         AsId asId = config.getAsId();
2076         if (asId == null) {
2077             LOG.error("bgp as-id is null");
2078             return replaySucceded;
2079         }
2080         long asNum = asId.getLocalAs();
2081         IpAddress routerId = asId.getRouterId();
2082         String rid = routerId == null ? "" : new String(routerId.getValue());
2083         int stalepathTime = (int) getStalePathtime(RESTART_DEFAULT_GR, config.getAsId());
2084         boolean announceFbit = true;
2085         boolean replayDone = false;
2086         final int numberOfStartBgpRetries = 3;
2087         RetryOnException startBgpRetry = new RetryOnException(numberOfStartBgpRetries);
2088         do {
2089             try {
2090                 LOG.debug("Replaying BGPConfig ");
2091                 br.startBgp(asNum, rid, stalepathTime, announceFbit);
2092                 LOG.debug("Replay BGPConfig successful");
2093                 replayDone = true;
2094                 break;
2095             } catch (BgpRouterException bre) {
2096                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
2097                     LOG.debug("Starting the routesync for exception", bre);
2098                     startBgpRetry.errorOccured();
2099                     if (!startBgpRetry.shouldRetry()) {
2100                         LOG.debug("starting route sync for BgpRouter exception");
2101                         doRouteSync();
2102                     }
2103                 } else {
2104                     LOG.error("Replay: startBgp() received exception error {} : ",
2105                             bre.getErrorCode(), bre);
2106                     startBgpRetry.errorOccured();
2107                 }
2108             } catch (TApplicationException tae) {
2109                 if (tae.getType() == BgpRouterException.BGP_ERR_ACTIVE) {
2110                     LOG.debug("Starting the routesync for exception", tae);
2111                     startBgpRetry.errorOccured();
2112                     if (!startBgpRetry.shouldRetry()) {
2113                         LOG.debug("starting route sync for Thrift BGP_ERR_ACTIVE exception");
2114                         doRouteSync();
2115                     }
2116                 } else if (tae.getType() == BgpRouterException.BGP_ERR_COMMON_FAILURE) {
2117                     LOG.debug("Starting the routesync for AS-ID started exception", tae);
2118                     startBgpRetry.errorOccured();
2119                     if (!startBgpRetry.shouldRetry()) {
2120                         LOG.debug("starting route sync for Thrift BGP_ERR_COMMON_FAILURE exception");
2121                         doRouteSync();
2122                     }
2123                 } else {
2124                     LOG.error("Replay: startBgp() received exception type {}: ",
2125                             tae.getType(), tae);
2126                     startBgpRetry.errorOccured();
2127                 }
2128             } catch (Exception e) {
2129                 //not unusual. We may have restarted & BGP is already on
2130                 LOG.error("Replay:startBgp() received exception: ", e);
2131                 startBgpRetry.errorOccured();
2132             }
2133         } while (startBgpRetry.shouldRetry());
2134
2135         replaySucceded = replayDone;
2136
2137         startBgpCountersTask();
2138         startBgpAlarmsTask();
2139
2140         /*
2141          * commenting this due to a bug with QBGP. Will uncomment once QBGP fix is done.
2142          * This wont have any functional impacts
2143          */
2144         //try {
2145         //    br.delayEOR(delayEorSeconds);
2146         //} catch (TException | BgpRouterException e) {
2147         //    LOG.error("Replay: delayEOR() number of seconds to wait for EOR from ODL:", e);
2148         //}
2149
2150         List<Neighbors> neighbors = config.getNeighbors();
2151         if (neighbors != null) {
2152             LOG.error("configuring existing Neighbors present for replay total neighbors {}", neighbors.size());
2153             boolean neighborConfigReplayResult = replayNbrConfig(neighbors, br);
2154             if (neighborConfigReplayResult == false) {
2155                 replaySucceded = false;
2156             }
2157         } else {
2158             LOG.error("no Neighbors present for replay config ");
2159         }
2160
2161         Logging logging = config.getLogging();
2162         if (logging != null) {
2163             try {
2164                 br.setLogging(logging.getFile(), logging.getLevel());
2165             } catch (TException | BgpRouterException e) {
2166                 LOG.error("Replay:setLogging() received exception", e);
2167             }
2168         }
2169
2170         GracefulRestart gracefulRestart = config.getGracefulRestart();
2171         if (gracefulRestart != null) {
2172             try {
2173                 br.addGracefulRestart(gracefulRestart.getStalepathTime().intValue());
2174             } catch (TException | BgpRouterException e) {
2175                 LOG.error("Replay:addGr() received exception", e);
2176             }
2177         }
2178
2179         List<Vrfs> vrfs = config.getVrfs();
2180         if (vrfs == null) {
2181             vrfs = new ArrayList<>();
2182         }
2183         for (Vrfs vrf : vrfs) {
2184             for (AddressFamiliesVrf adf : vrf.getAddressFamiliesVrf()) {
2185                 try {
2186                     br.addVrf(BgpUtil.getLayerType(adf), vrf.getRd(), vrf.getImportRts(),
2187                             vrf.getExportRts());
2188                 } catch (TException | BgpRouterException e) {
2189                     LOG.error("Replay:addVrf() received exception", e);
2190                 }
2191             }
2192         }
2193
2194         List<Networks> ln = config.getNetworks();
2195         if (ln != null) {
2196             for (Networks net : ln) {
2197                 String rd = net.getRd();
2198                 String pfxlen = net.getPrefixLen();
2199                 String nh = net.getNexthop().getValue();
2200                 Long label = net.getLabel();
2201                 int lbl = label == null ? 0 : label.intValue();
2202                 int l3vni = net.getL3vni() == null ? 0 : net.getL3vni().intValue();
2203                 int l2vni = net.getL2vni() == null ? 0 : net.getL2vni().intValue();
2204                 if (rd == null && lbl > 0) {
2205                     //LU prefix is being deleted.
2206                     rd = Integer.toString(lbl);
2207                 }
2208
2209                 BgpControlPlaneType protocolType = net.getBgpControlPlaneType();
2210                 int ethernetTag = net.getEthtag().intValue();
2211                 String esi = net.getEsi();
2212                 String macaddress = net.getMacaddress();
2213                 EncapType encapType = net.getEncapType();
2214                 String routerMac = net.getRoutermac();
2215
2216                 try {
2217                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni,
2218                             BgpUtil.convertToThriftProtocolType(protocolType),
2219                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
2220                 } catch (TException | BgpRouterException e) {
2221                     LOG.error("Replay:addPfx() received exception", e);
2222                 }
2223             }
2224         }
2225
2226
2227         List<Multipath> multipaths = config.getMultipath();
2228
2229         if (multipaths != null) {
2230             for (Multipath multipath : multipaths) {
2231                 if (multipath != null) {
2232                     af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
2233                     af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
2234
2235                     try {
2236                         if (multipath.isMultipathEnabled()) {
2237                             br.enableMultipath(afi, safi);
2238                         } else {
2239                             br.disableMultipath(afi, safi);
2240                         }
2241                     } catch (TException | BgpRouterException e) {
2242                         LOG.info("Replay:multipaths() received exception", e);
2243                     }
2244                 }
2245             }
2246         }
2247         List<VrfMaxpath> vrfMaxpaths = config.getVrfMaxpath();
2248         if (vrfMaxpaths != null) {
2249             for (VrfMaxpath vrfMaxpath : vrfMaxpaths) {
2250                 try {
2251                     br.multipaths(vrfMaxpath.getRd(), vrfMaxpath.getMaxpaths());
2252                 } catch (TException | BgpRouterException e) {
2253                     LOG.info("Replay:vrfMaxPath() received exception", e);
2254                 }
2255             }
2256         }
2257
2258         //send End of Rib Marker to Qthriftd.
2259         final int numberOfEORRetries = 3;
2260         replayDone = false;
2261         RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
2262         do {
2263             try {
2264                 br.sendEOR();
2265                 LOG.debug("Replay sendEOR {} successful");
2266                 replayDone = true;
2267                 break;
2268             } catch (Exception e) {
2269                 eorRetry.errorOccured();
2270                 LOG.error("Replay:sedEOR() received exception:", e);
2271             }
2272         } while (eorRetry.shouldRetry());
2273
2274         return replaySucceded && replayDone;
2275     }
2276
2277     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
2278         bgpUtil.update(iid, dto);
2279     }
2280
2281     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
2282         bgpUtil.delete(iid);
2283     }
2284
2285     public void startConfig(String bgpHost, int thriftPort) {
2286         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2287                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2288         InstanceIdentifier<ConfigServer> iid = iib.build();
2289         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
2290         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
2291                 .setPort((long) thriftPort).build();
2292         update(iid, dto);
2293     }
2294
2295     public void startBgp(long as, String routerId, int spt, boolean fbit) {
2296         IpAddress rid = routerId == null ? null : new IpAddress(routerId.toCharArray());
2297         Long staleTime = (long) spt;
2298         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2299                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2300         InstanceIdentifier<AsId> iid = iib.build();
2301         AsId dto = new AsIdBuilder().setLocalAs(as)
2302                 .setRouterId(rid)
2303                 .setStalepathTime(staleTime)
2304                 .setAnnounceFbit(fbit).build();
2305         update(iid, dto);
2306     }
2307
2308     public void addLogging(String fileName, String logLevel) {
2309         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2310                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2311         InstanceIdentifier<Logging> iid = iib.build();
2312         Logging dto = new LoggingBuilder().setFile(fileName)
2313                 .setLevel(logLevel).build();
2314         update(iid, dto);
2315     }
2316
2317     public void addGracefulRestart(int staleTime) {
2318         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2319                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
2320         InstanceIdentifier<GracefulRestart> iid = iib.build();
2321         GracefulRestart dto = new GracefulRestartBuilder()
2322                 .setStalepathTime((long) staleTime).build();
2323         update(iid, dto);
2324     }
2325
2326     public void addNeighbor(
2327             String nbrIp, long remoteAs, @Nullable final TcpMd5SignaturePasswordType md5Secret) {
2328         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2329         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2330                 InstanceIdentifier.builder(Bgp.class)
2331                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2332         InstanceIdentifier<Neighbors> iid = iib.build();
2333         TcpSecurityOption tcpSecOption = null;
2334         if (md5Secret != null) {
2335             tcpSecOption = new TcpMd5SignatureOptionBuilder().setTcpMd5SignaturePassword(md5Secret).build();
2336         } // else let tcpSecOption be null
2337         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
2338                 .setRemoteAs(remoteAs).setTcpSecurityOption(tcpSecOption).build();
2339         update(iid, dto);
2340     } // public addNeighbor(nbrIp, remoteAs, md5Secret)
2341
2342     public void addUpdateSource(String nbrIp, String srcIp) {
2343         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2344         Ipv4Address srcAddr = new Ipv4Address(srcIp);
2345         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2346                 InstanceIdentifier.builder(Bgp.class)
2347                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2348                         .child(UpdateSource.class);
2349         InstanceIdentifier<UpdateSource> iid = iib.build();
2350         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
2351                 .setSourceIp(srcAddr).build();
2352         update(iid, dto);
2353     }
2354
2355     public void addEbgpMultihop(String nbrIp, int hops) {
2356         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2357         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2358                 InstanceIdentifier.builder(Bgp.class)
2359                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2360                         .child(EbgpMultihop.class);
2361         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2362         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
2363                 .setNhops((long) hops).build();
2364         update(iid, dto);
2365     }
2366
2367     public void addAddressFamily(String nbrIp, int afi, int safi) {
2368         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2369         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2370                 InstanceIdentifier.builder(Bgp.class)
2371                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2372                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2373         InstanceIdentifier<AddressFamilies> iid = iib.build();
2374         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
2375                 .setAfi((long) afi).setSafi((long) safi).build();
2376         update(iid, dto);
2377     }
2378
2379     public void addPrefix(String rd, String macAddress, String pfx, List<String> nhList,
2380               VrfEntry.EncapType encapType, long lbl, long l3vni, long l2vni, String gatewayMac) {
2381         for (String nh : nhList) {
2382             Ipv4Address nexthop = nh != null ? new Ipv4Address(nh) : null;
2383             Long label = lbl;
2384             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
2385                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
2386             NetworksBuilder networksBuilder = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
2387                                                 .setLabel(label).setEthtag(BgpConstants.DEFAULT_ETH_TAG);
2388             buildVpnEncapSpecificInfo(networksBuilder, encapType, label, l3vni, l2vni, macAddress, gatewayMac);
2389             update(iid, networksBuilder.build());
2390         }
2391     }
2392
2393     private static void buildVpnEncapSpecificInfo(NetworksBuilder builder, VrfEntry.EncapType encapType, long label,
2394                                                   long l3vni, long l2vni, String macAddress, String gatewayMac) {
2395         if (encapType.equals(VrfEntry.EncapType.Mplsgre)) {
2396             builder.setLabel(label).setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLL3VPN)
2397                     .setEncapType(EncapType.GRE);
2398         } else {
2399             builder.setL3vni(l3vni).setL2vni(l2vni).setMacaddress(macAddress).setRoutermac(gatewayMac)
2400                     .setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLEVPN).setEncapType(EncapType.VXLAN);
2401         }
2402     }
2403
2404     // TODO: add LayerType as arg - supports command
2405     public void addVrf(String rd, List<String> irts, List<String> erts, AddressFamily addressFamily) {
2406         Vrfs vrf = bgpUtil.getVrfFromRd(rd);
2407         List<AddressFamiliesVrf> adfList = new ArrayList<>(1);
2408         if (vrf != null) {
2409             adfList = vrf.getAddressFamiliesVrf();
2410         }
2411         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2412         if (addressFamily.equals(AddressFamily.IPV4)) {
2413             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2414             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2415         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2416             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2417             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2418         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2419             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2420             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2421         }
2422         AddressFamiliesVrf adf = adfBuilder.build();
2423         adfList.add(adf);
2424         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib = InstanceIdentifier.builder(Bgp.class)
2425                 .child(Vrfs.class, new VrfsKey(rd));
2426         InstanceIdentifier<Vrfs> iid = iib.build();
2427         Vrfs dto = new VrfsBuilder().setRd(rd).setImportRts(irts)
2428             .setExportRts(erts).setAddressFamiliesVrf(adfList).build();
2429
2430         List<AddressFamiliesVrf> listAdFamilies = mapNewAdFamily.get(rd);
2431         if (listAdFamilies != null) {
2432             listAdFamilies.add(adf);
2433         } else {
2434             mapNewAdFamily.put(rd, adfList);
2435         }
2436
2437         try {
2438             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
2439         } catch (TransactionCommitFailedException e) {
2440             LOG.error("Error adding VRF to datastore", e);
2441             throw new RuntimeException(e);
2442         }
2443     }
2444
2445     public void stopConfig() {
2446         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2447                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2448         InstanceIdentifier<ConfigServer> iid = iib.build();
2449         delete(iid);
2450     }
2451
2452     public void stopBgp() {
2453         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2454                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2455         InstanceIdentifier<AsId> iid = iib.build();
2456         delete(iid);
2457     }
2458
2459     public void delLogging() {
2460         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2461                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2462         InstanceIdentifier<Logging> iid = iib.build();
2463         delete(iid);
2464     }
2465
2466     public void delGracefulRestart() {
2467         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2468                 InstanceIdentifier.builder(Bgp.class)
2469                         .child(GracefulRestart.class);
2470         InstanceIdentifier<GracefulRestart> iid = iib.build();
2471         delete(iid);
2472     }
2473
2474     public void delNeighbor(String nbrIp) {
2475         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2476         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2477                 InstanceIdentifier.builder(Bgp.class)
2478                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2479         InstanceIdentifier<Neighbors> iid = iib.build();
2480         delete(iid);
2481     }
2482
2483     public void delUpdateSource(String nbrIp) {
2484         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2485         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2486                 InstanceIdentifier.builder(Bgp.class)
2487                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2488                         .child(UpdateSource.class);
2489         InstanceIdentifier<UpdateSource> iid = iib.build();
2490         delete(iid);
2491     }
2492
2493     public void delEbgpMultihop(String nbrIp) {
2494         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2495         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2496                 InstanceIdentifier.builder(Bgp.class)
2497                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2498                         .child(EbgpMultihop.class);
2499         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2500         delete(iid);
2501     }
2502
2503     public void delAddressFamily(String nbrIp, int afi, int safi) {
2504         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2505         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2506                 InstanceIdentifier.builder(Bgp.class)
2507                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2508                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2509         InstanceIdentifier<AddressFamilies> iid = iib.build();
2510         delete(iid);
2511     }
2512
2513     public void delPrefix(String rd, String pfx) {
2514         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
2515                 InstanceIdentifier.builder(Bgp.class)
2516                         .child(Networks.class, new NetworksKey(pfx, rd));
2517         InstanceIdentifier<Networks> iid = iib.build();
2518         delete(iid);
2519     }
2520
2521     public boolean delVrf(String rd, AddressFamily addressFamily) {
2522         if (addressFamily == null) {
2523             LOG.error("delVrf: vrf {}, addressFamily invalid", rd);
2524             return false;
2525         }
2526         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2527         if (addressFamily.equals(AddressFamily.IPV4)) {
2528             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2529             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2530         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2531             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2532             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2533         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2534             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2535             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2536         }
2537         Vrfs vrfOriginal = bgpUtil.getVrfFromRd(rd);
2538         if (vrfOriginal == null) {
2539             LOG.error("delVrf: no vrf with existing rd {}. step aborted", rd);
2540             return false;
2541         }
2542
2543         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2544                 InstanceIdentifier.builder(Bgp.class)
2545                         .child(Vrfs.class, new VrfsKey(rd));
2546
2547         InstanceIdentifier<Vrfs> iid = iib.build();
2548
2549         @SuppressWarnings("static-access")
2550         InstanceIdentifier<Bgp> iid6 =  iid.builder(Bgp.class).build()
2551                 .child(Multipath.class, new MultipathKey(adfBuilder.getAfi(), adfBuilder.getSafi())).create(Bgp.class);
2552         InstanceIdentifierBuilder<Vrfs> iib3 = iid6.child(Vrfs.class, new VrfsKey(rd)).builder();
2553         InstanceIdentifier<Vrfs> iidFinal = iib3.build();
2554
2555         //** update or delete the vrfs with the rest of AddressFamilies already present in the last list
2556         AddressFamiliesVrf adfToDel = adfBuilder.build();
2557         List<AddressFamiliesVrf> adfListOriginal = vrfOriginal.getAddressFamiliesVrf() == null
2558                 ? new ArrayList<>() : vrfOriginal.getAddressFamiliesVrf();
2559         List<AddressFamiliesVrf> adfListToRemoveFromOriginal = new ArrayList<>();
2560         adfListOriginal.forEach(adf -> {
2561             if (adf.equals(adfToDel)) {
2562                 adfListToRemoveFromOriginal.add(adfToDel);
2563                 return;
2564             }
2565         });
2566         for (AddressFamiliesVrf adfToRemove : adfListToRemoveFromOriginal) {
2567             adfListOriginal.remove(adfToRemove);
2568             try {
2569                 SingleTransactionDataBroker.syncWrite(dataBroker,
2570                         LogicalDatastoreType.CONFIGURATION, iid, vrfOriginal);
2571             } catch (TransactionCommitFailedException e) {
2572                 LOG.error("delVrf: Error updating VRF to datastore", e);
2573                 throw new RuntimeException(e);
2574             }
2575         }
2576         if (adfListOriginal.isEmpty()) {
2577             delete(iidFinal);
2578             return true;
2579         }
2580         // not all is removed
2581         return false;
2582     }
2583
2584     public void setMultipathStatus(af_afi afi, af_safi safi, boolean enable) {
2585         long lafi = afi.getValue();
2586         long lsafi = safi.getValue();
2587
2588         InstanceIdentifier.InstanceIdentifierBuilder<Multipath> iib =
2589                 InstanceIdentifier
2590                         .builder(Bgp.class)
2591                         .child(Multipath.class,
2592                                 new MultipathKey(Long.valueOf(afi.getValue()), Long.valueOf(safi.getValue())));
2593
2594         Multipath dto = new MultipathBuilder().setAfi(lafi).setSafi(lsafi).setMultipathEnabled(enable).build();
2595         update(iib.build(), dto);
2596     }
2597
2598     public void multipaths(String rd, int maxpath) {
2599         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
2600                 InstanceIdentifier
2601                         .builder(Bgp.class)
2602                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
2603
2604         VrfMaxpath dto = new VrfMaxpathBuilder().setRd(rd).setMaxpaths(maxpath).build();
2605         update(iib.build(), dto);
2606     }
2607
2608     /*
2609     * Remove Stale Marked Routes after timer expiry.
2610     */
2611     private class RouteCleanup implements Callable<Integer> {
2612
2613         @Override
2614         public Integer call() {
2615             totalCleared = 0;
2616             try {
2617                 if (staledFibEntriesMap.isEmpty()) {
2618                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
2619                 } else {
2620                     for (String rd : staledFibEntriesMap.keySet()) {
2621                         if (Thread.interrupted()) {
2622                             return 0;
2623                         }
2624                         Map<String, Long> map = staledFibEntriesMap.get(rd);
2625                         if (map != null) {
2626                             for (String key : map.keySet()) {
2627                                 if (Thread.interrupted()) {
2628                                     return 0;
2629                                 }
2630                                 String prefix = extractPrefix(key);
2631                                 String nextHop = extractNextHop(key);
2632                                 totalCleared++;
2633                                 LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}, nextHop:{}",
2634                                         rd, prefix, nextHop);
2635                                 fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix, nextHop);
2636                             }
2637                         }
2638                     }
2639                 }
2640             } finally {
2641                 staledFibEntriesMap.clear();
2642             }
2643             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
2644             return 0;
2645         }
2646     }
2647
2648     /*
2649      * BGP restart scenario, ODL-BGP manager was/is running.
2650      * On re-sync notification, Get a copy of FIB database.
2651      */
2652     public void createStaleFibMap() {
2653         totalStaledCount = 0;
2654         try {
2655             staledFibEntriesMap.clear();
2656             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2657
2658             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2659                     LogicalDatastoreType.CONFIGURATION, id);
2660             if (fibEntries.isPresent()) {
2661                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2662                 for (VrfTables vrfTable : staleVrfTables) {
2663                     Map<String, Long> staleFibEntMap = new HashMap<>();
2664                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2665                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2666                             //Stale marking and cleanup is only meant for the routes learned through BGP.
2667                             continue;
2668                         }
2669                         if (Thread.interrupted()) {
2670                             break;
2671                         }
2672                         totalStaledCount++;
2673                         //Create MAP from staleVrfTables.
2674                         vrfEntry.getRoutePaths()
2675                                 .forEach(
2676                                     routePath -> staleFibEntMap.put(
2677                                             appendNextHopToPrefix(vrfEntry.getDestPrefix(),
2678                                                     routePath.getNexthopAddress()), routePath.getLabel()));
2679                     }
2680                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), staleFibEntMap);
2681                 }
2682             } else {
2683                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
2684             }
2685         } catch (ReadFailedException e) {
2686             LOG.error("createStaleFibMap:: error ", e);
2687         }
2688         LOG.error("created {} staled entries ", totalStaledCount);
2689     }
2690
2691     /*
2692      * BGP config remove scenario, Need to remove all the
2693      * external routes from FIB.
2694      */
2695     public void deleteExternalFibRoutes() {
2696         totalExternalRoutes = 0;
2697         totalExternalMacRoutes = 0;
2698         try {
2699             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2700
2701             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2702                     LogicalDatastoreType.CONFIGURATION, id);
2703             if (fibEntries.isPresent()) {
2704                 if (fibEntries.get().getVrfTables() == null) {
2705                     LOG.error("deleteExternalFibRoutes::getVrfTables is null");
2706                     return;
2707                 }
2708                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2709                 for (VrfTables vrfTable : staleVrfTables) {
2710                     String rd = vrfTable.getRouteDistinguisher();
2711                     if (vrfTable.getVrfEntry() != null) {
2712                         for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2713                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2714                                 //route cleanup is only meant for the routes learned through BGP.
2715                                 continue;
2716                             }
2717                             totalExternalRoutes++;
2718                             fibDSWriter.removeFibEntryFromDS(rd, vrfEntry.getDestPrefix());
2719                         }
2720                     } else if (vrfTable.getMacVrfEntry() != null) {
2721                         for (MacVrfEntry macEntry : vrfTable.getMacVrfEntry()) {
2722                             if (RouteOrigin.value(macEntry.getOrigin()) != RouteOrigin.BGP) {
2723                                 //route cleanup is only meant for the routes learned through BGP.
2724                                 continue;
2725                             }
2726                             totalExternalMacRoutes++;
2727                             fibDSWriter.removeMacEntryFromDS(rd, macEntry.getMac());
2728                         }
2729                     }
2730                 }
2731             } else {
2732                 LOG.error("deleteExternalFibRoutes:: FIBentries.class is not present");
2733             }
2734         } catch (ReadFailedException e) {
2735             LOG.error("deleteExternalFibRoutes:: error ", e);
2736         }
2737         LOG.debug("deleted {} fib entries {} mac entries", totalExternalRoutes, totalExternalMacRoutes);
2738     }
2739
2740     public boolean addToRt2TepMap(String rd, String tepIp, String mac, Long l2vni) {
2741         boolean isFirstMacUpdateFromTep = false;
2742         if (rt2TepMap.containsKey(rd)) {
2743             if (rt2TepMap.get(rd).containsKey(tepIp)) {
2744                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and tep-ip {}. No Elan DS write required",
2745                         mac, l2vni, rd, tepIp);
2746                 rt2TepMap.get(rd).get(tepIp).put(mac, l2vni);
2747             } else {
2748                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and new tep-ip {}",
2749                         mac, l2vni, rd, tepIp);
2750                 isFirstMacUpdateFromTep = true;
2751                 Map<String, Long> macList = new HashMap<>();
2752                 macList.put(mac, l2vni);
2753                 rt2TepMap.get(rd).put(tepIp, macList);
2754             }
2755         } else {
2756             LOG.debug("RT2 with mac {} l2vni {} from new rd {} and tep ip {}",
2757                     mac, l2vni, rd, tepIp);
2758             isFirstMacUpdateFromTep = true;
2759             Map<String, Long> macList = new HashMap<>();
2760             macList.put(mac, l2vni);
2761             Map<String, Map<String, Long>> tepIpMacMap = new HashMap<>();
2762             tepIpMacMap.put(tepIp, macList);
2763             rt2TepMap.put(rd, tepIpMacMap);
2764         }
2765         return isFirstMacUpdateFromTep;
2766     }
2767
2768     public boolean deleteFromRt2TepMap(String rd, String tepIp, String mac) {
2769         boolean isLastMacUpdateFromTep = false;
2770         LOG.debug("RT2 withdraw with rd {} mac {} tep-ip {} ", rd, mac, tepIp);
2771         if (rt2TepMap.containsKey(rd)) {
2772             if (rt2TepMap.get(rd).containsKey(tepIp)) {
2773                 if (rt2TepMap.get(rd).get(tepIp).containsKey(mac)) {
2774                     LOG.debug("RT2 Withdraw : Removing the mac {} from Map", mac);
2775                     rt2TepMap.get(rd).get(tepIp).remove(mac);
2776                     if (rt2TepMap.get(rd).get(tepIp).isEmpty()) {
2777                         isLastMacUpdateFromTep = true;
2778                         LOG.debug("RT2 Withdraw : Removing the tep-ip {} from Map", tepIp);
2779                         rt2TepMap.get(rd).remove(tepIp);
2780                         if (rt2TepMap.get(rd).isEmpty()) {
2781                             LOG.debug("RT2 Withdraw : Removing the rd {} from Map", rd);
2782                             rt2TepMap.remove(rd);
2783                         }
2784                     }
2785                 }
2786             }
2787         }
2788         return isLastMacUpdateFromTep;
2789     }
2790
2791     public Collection<String> getTepIPs(String rd) {
2792         final Map<String, Map<String, Long>> tepIpMap = rt2TepMap.get(rd);
2793         return tepIpMap != null ? tepIpMap.keySet() : Collections.emptyList();
2794     }
2795
2796     public boolean isBgpConnected() {
2797         return bgpRouter.isBgpConnected();
2798     }
2799
2800     public long getLastConnectedTS() {
2801         return bgpRouter.getLastConnectedTS();
2802     }
2803
2804     public long getConnectTS() {
2805         return bgpRouter.getConnectTS();
2806     }
2807
2808     public long getStartTS() {
2809         return bgpRouter.getStartTS();
2810     }
2811
2812     public TTransport getTransport() {
2813         return bgpRouter.getTransport();
2814     }
2815
2816     public int getTotalStaledCount() {
2817         return totalStaledCount;
2818     }
2819
2820     public int getTotalCleared() {
2821         return totalCleared;
2822     }
2823
2824     public BgpCounters getBgpCounters() {
2825         return bgpCountersReference.get();
2826     }
2827
2828     private void startBgpCountersTask() {
2829         if (getBgpCounters() == null && bgpCountersReference.compareAndSet(null,
2830                 new BgpCounters(getBgpSdncMipIp(), metricProvider))) {
2831             bgpCountersTask = executor.scheduleAtFixedRate(bgpCountersReference.get(), 0, 120 * 1000,
2832                     TimeUnit.MILLISECONDS);
2833             LOG.info("Bgp Counters task scheduled for every two minutes.");
2834
2835             bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
2836         }
2837     }
2838
2839     private void stopBgpCountersTask() {
2840         final BgpCounters bgpCounters = bgpCountersReference.getAndSet(null);
2841         if (bgpCounters != null) {
2842             bgpCountersTask.cancel(true);
2843             bgpCounters.close();
2844         }
2845     }
2846
2847     private void startBgpAlarmsTask() {
2848         if (getBgpAlarms() == null && bgpAlarmsReference.compareAndSet(null, new BgpAlarms(this))) {
2849             bgpAlarmsReference.get().init();
2850             bgpAlarmsTask = executor.scheduleAtFixedRate(bgpAlarmsReference.get(), 0, 60 * 1000, TimeUnit.MILLISECONDS);
2851             LOG.info("Bgp Alarms task scheduled for every minute.");
2852         }
2853     }
2854
2855     private void stopBgpAlarmsTask() {
2856         final BgpAlarms bgpAlarms = bgpAlarmsReference.getAndSet(null);
2857         if (bgpAlarms != null) {
2858             bgpAlarmsTask.cancel(true);
2859             bgpAlarms.close();
2860         }
2861     }
2862
2863     public BgpAlarms getBgpAlarms() {
2864         return bgpAlarmsReference.get();
2865     }
2866
2867     private static String appendNextHopToPrefix(String prefix, String nextHop) {
2868         return prefix + ":" + nextHop;
2869     }
2870
2871     private static String extractPrefix(String prefixNextHop) {
2872         return prefixNextHop.split(":")[0];
2873     }
2874
2875     private static String extractNextHop(String prefixNextHop) {
2876         return prefixNextHop.split(":")[1];
2877     }
2878
2879     private static String extractMd5Secret(final Neighbors val) {
2880         String md5Secret = null;
2881         TcpSecurityOption tcpSecOpt = val.getTcpSecurityOption();
2882         if (tcpSecOpt != null) {
2883             if (tcpSecOpt instanceof TcpMd5SignatureOption) {
2884                 md5Secret = ((TcpMd5SignatureOption) tcpSecOpt).getTcpMd5SignaturePassword().getValue();
2885             } else { // unknown TcpSecurityOption
2886                 LOG.debug("neighbors  Ignored unknown tcp-security-option of peer {}", val.getAddress().getValue());
2887             }
2888         }
2889         return md5Secret;
2890     } // private method extractMd5Secret
2891 }