bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.net.InetAddresses;
  13 import com.google.common.util.concurrent.ThreadFactoryBuilder;
  14 import io.netty.util.concurrent.GlobalEventExecutor;
  15 import java.io.IOException;
  16 import java.lang.reflect.Constructor;
  17 import java.lang.reflect.InvocationTargetException;
  18 import java.net.InetAddress;
  19 import java.net.InetSocketAddress;
  20 import java.net.NetworkInterface;
  21 import java.net.SocketException;
  22 import java.util.ArrayList;
  23 import java.util.Collection;
  24 import java.util.Collections;
  25 import java.util.Enumeration;
  26 import java.util.HashMap;
  27 import java.util.Iterator;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Timer;
  32 import java.util.concurrent.Callable;
  33 import java.util.concurrent.ConcurrentHashMap;
  34 import java.util.concurrent.CountDownLatch;
  35 import java.util.concurrent.Executors;
  36 import java.util.concurrent.Future;
  37 import java.util.concurrent.ScheduledExecutorService;
  38 import java.util.concurrent.ScheduledFuture;
  39 import java.util.concurrent.TimeUnit;
  40 import java.util.concurrent.atomic.AtomicInteger;
  41 import java.util.concurrent.atomic.AtomicReference;
  42 import javax.annotation.Nullable;
  43 import javax.annotation.PreDestroy;
  44 import javax.inject.Inject;
  45 import javax.inject.Singleton;
  46 import org.apache.thrift.TApplicationException;
  47 import org.apache.thrift.TException;
  48 import org.apache.thrift.transport.TTransport;
  49 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  50 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  51 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  52 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  53 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  54 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  55 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  56 import org.opendaylight.genius.mdsalutil.NwConstants;
  57 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  58 import org.opendaylight.infrautils.metrics.MetricProvider;
  59 import org.opendaylight.mdsal.eos.binding.api.Entity;
  60 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipCandidateRegistration;
  61 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipListenerRegistration;
  62 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  63 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  64 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  65 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  66 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  67 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  68 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  69 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  70 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  71 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  72 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  73 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  74 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  75 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  76 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  77 import org.opendaylight.netvirt.bgpmanager.thrift.gen.protocol_type;
  78 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  79 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  80 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  81 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  82 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.AddressFamily;
  83 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  84 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
  85 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EncapType;
  86 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.TcpMd5SignaturePasswordType;
  87 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
  88 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
  90 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
  92 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
  94 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Multipath;
  96 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathKey;
  98 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Neighbors;
  99 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsKey;
 101 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Networks;
 102 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksKey;
 104 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpath;
 105 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathKey;
 107 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Vrfs;
 108 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsKey;
 110 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamilies;
 111 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesKey;
 113 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihop;
 114 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihopBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSource;
 116 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSourceBuilder;
 117 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfs.AddressFamiliesVrf;
 118 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfs.AddressFamiliesVrfBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.TcpSecurityOption;
 120 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOption;
 121 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOptionBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
 123 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.macvrfentries.MacVrfEntry;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 130 import org.opendaylight.yangtools.yang.binding.DataObject;
 131 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 132 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 133 import org.osgi.framework.BundleContext;
 134 import org.osgi.util.tracker.ServiceTracker;
 135 import org.slf4j.Logger;
 136 import org.slf4j.LoggerFactory;
 137
 138 @Singleton
 139 public class BgpConfigurationManager {
 140     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 141
 142     // to have stale FIB map (RD, Prefix)
 143     //  number of seconds wait for route sync-up between ODL and BGP
 144     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 145     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 146     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 147     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 148     private static final String DEF_UPORT = "6644";
 149     private static final String DEF_CHOST = "255.255.255.255"; // Invalid Host IP
 150     private static final String DEF_CPORT = "7644";
 151     private static final String DEF_BGP_SDNC_MIP = "127.0.0.1";
 152     private static final String BGP_SDNC_MIP = "vpnservice.bgp.thrift.sdnc.mip";
 153     private static final int RESTART_DEFAULT_GR = 90;
 154     private static final int DS_RETRY_COUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
 155     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
 156     private static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 157     private static final String BGP_EOR_DELAY = "vpnservice.bgp.eordelay";
 158     private static final String DEF_BGP_EOR_DELAY = "1800";
 159     private static final String BGP_ENTITY_NAME = "bgp";
 160     private static final String ADD_WARN = "Config store updated; undo with Delete if needed.";
 161     private static final String DEL_WARN = "Config store updated; undo with Add if needed.";
 162     private static final String UPD_WARN = "Update operation not supported; Config store updated;"
 163             + " restore with another Update if needed.";
 164
 165     private static final Class<?>[] REACTORS = {
 166         ConfigServerReactor.class, AsIdReactor.class,
 167         GracefulRestartReactor.class, LoggingReactor.class,
 168         NeighborsReactor.class, UpdateSourceReactor.class,
 169         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 170         NetworksReactor.class, VrfsReactor.class, BgpReactor.class,
 171         MultipathReactor.class, VrfMaxpathReactor.class
 172     };
 173
 174     private IBgpManager bgpManager;
 175     private final DataBroker dataBroker;
 176     private final FibDSWriter fibDSWriter;
 177     private final IVpnLinkService vpnLinkService;
 178     private final BundleContext bundleContext;
 179     private final BgpUtil bgpUtil;
 180     private volatile Bgp config;
 181     private final BgpRouter bgpRouter;
 182     private final BgpSyncHandle bgpSyncHandle = new BgpSyncHandle();
 183     private volatile BgpThriftService bgpThriftService = null;
 184     private final int delayEorSeconds;
 185
 186     private final CountDownLatch initer = new CountDownLatch(1);
 187
 188     private final String hostStartup;
 189     private final String portStartup;
 190
 191     private final AtomicReference<BgpCounters> bgpCountersReference = new AtomicReference<>();
 192     private ScheduledFuture<?> bgpCountersTask;
 193
 194     private final AtomicReference<BgpAlarms> bgpAlarmsReference = new AtomicReference<>();
 195     private ScheduledFuture<?> bgpAlarmsTask;
 196
 197     private Future<?> lastReplayJobFt;
 198     private ScheduledFuture<?> routeCleanupFuture;
 199
 200     private long staleStartTime;
 201     private long staleEndTime;
 202     private long cfgReplayStartTime;
 203     private long cfgReplayEndTime;
 204     private long staleCleanupTime;
 205     private int totalStaledCount;
 206     private int totalCleared;
 207     private int totalExternalRoutes;
 208     private int totalExternalMacRoutes;
 209
 210     private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(
 211             new ThreadFactoryBuilder().setNameFormat("BgpConfigurationManager-%d").setDaemon(true).build());
 212
 213     /**
 214      * this map store the new address families to send to quagga. When it is sended you must clear it.
 215      * The keys String are rd (route distinguisher).
 216      * */
 217     private final ConcurrentHashMap<String, List<AddressFamiliesVrf>> mapNewAdFamily = new ConcurrentHashMap<>();
 218
 219     // map<rd, map<prefix/len:nexthop, label>>
 220     private final Map<String, Map<String, Long>> staledFibEntriesMap = new ConcurrentHashMap<>();
 221
 222     // map<rd, map<tep-ip, map<mac, l2vni>>>
 223     private final Map<String, Map<String, Map<String, Long>>> rt2TepMap = new ConcurrentHashMap<>();
 224
 225     private final List<AutoCloseable> listeners = new ArrayList<>();
 226
 227     private final EntityOwnershipUtils entityOwnershipUtils;
 228     private final EntityOwnershipCandidateRegistration candidateRegistration;
 229     private final EntityOwnershipListenerRegistration entityListenerRegistration;
 230     private final MetricProvider metricProvider;
 231
 232     @Inject
 233     public BgpConfigurationManager(final DataBroker dataBroker,
 234             final EntityOwnershipService entityOwnershipService,
 235             final FibDSWriter fibDSWriter,
 236             final IVpnLinkService vpnLinkSrvce,
 237             final BundleContext bundleContext,
 238             final BgpUtil bgpUtil,
 239             final MetricProvider metricProvider) {
 240         this.dataBroker = dataBroker;
 241         this.fibDSWriter = fibDSWriter;
 242         this.vpnLinkService = vpnLinkSrvce;
 243         this.bundleContext = bundleContext;
 244         this.bgpUtil = bgpUtil;
 245         this.metricProvider = metricProvider;
 246         String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
 247         hostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 248         portStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 249         LOG.info("ConfigServer at {}:{}", hostStartup, portStartup);
 250         VtyshCli.setHostAddr(hostStartup);
 251         ClearBgpCli.setHostAddr(hostStartup);
 252         bgpRouter = BgpRouter.newInstance(this::getConfig, this::isBGPEntityOwner);
 253         delayEorSeconds = Integer.parseInt(getProperty(BGP_EOR_DELAY, DEF_BGP_EOR_DELAY));
 254         registerCallbacks();
 255
 256         entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 257
 258         candidateRegistration = registerEntityCandidate(entityOwnershipService);
 259         entityListenerRegistration = registerEntityListener(entityOwnershipService);
 260
 261         LOG.info("BGP Configuration manager initialized");
 262         initer.countDown();
 263
 264         GlobalEventExecutor.INSTANCE.execute(() -> {
 265             ServiceTracker<IBgpManager, ?> tracker = null;
 266             try {
 267                 tracker = new ServiceTracker<>(bundleContext, IBgpManager.class, null);
 268                 tracker.open();
 269                 bgpManager = (IBgpManager) tracker.waitForService(TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES));
 270                 Preconditions.checkState(bgpManager != null, "IBgpManager service not found");
 271             } catch (InterruptedException e) {
 272                 throw new IllegalStateException("Error retrieving IBgpManager service", e);
 273             } finally {
 274                 if (tracker != null) {
 275                     tracker.close();
 276                 }
 277             }
 278
 279             if (InetAddresses.isInetAddress(getBgpSdncMipIp())) {
 280                 InetSocketAddress bgpThriftServerSocketAddr = new InetSocketAddress(getBgpSdncMipIp(),
 281                         Integer.parseInt(updatePort));
 282                 bgpThriftService = new BgpThriftService(bgpThriftServerSocketAddr, bgpManager, this);
 283                 if (isBGPEntityOwner()) {
 284                     //I am EoS owner of BGP, opening bgp thrift UPDATE-SERVER port.
 285                     LOG.info("BGP Configuration manager initialized: UPDATE-SERVER started");
 286                     bgpThriftService.start();
 287                 }
 288                 LOG.info("UPDATE server started :ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 289             } else {
 290                 LOG.error("Failed to init UPDATE server invalid ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 291             }
 292             LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 293         });
 294     }
 295
 296     public String getBgpSdncMipIp() {
 297         return getProperty(BGP_SDNC_MIP, DEF_BGP_SDNC_MIP);
 298     }
 299
 300     public long getStaleCleanupTime() {
 301         return staleCleanupTime;
 302     }
 303
 304     public void setStaleCleanupTime(long staleCleanupTime) {
 305         this.staleCleanupTime = staleCleanupTime;
 306     }
 307
 308     public long getCfgReplayEndTime() {
 309         return cfgReplayEndTime;
 310     }
 311
 312     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 313         this.cfgReplayEndTime = cfgReplayEndTime;
 314     }
 315
 316     public long getCfgReplayStartTime() {
 317         return cfgReplayStartTime;
 318     }
 319
 320     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 321         this.cfgReplayStartTime = cfgReplayStartTime;
 322     }
 323
 324     public long getStaleEndTime() {
 325         return staleEndTime;
 326     }
 327
 328     public void setStaleEndTime(long staleEndTime) {
 329         this.staleEndTime = staleEndTime;
 330     }
 331
 332     public long getStaleStartTime() {
 333         return staleStartTime;
 334     }
 335
 336     public void setStaleStartTime(long staleStartTime) {
 337         this.staleStartTime = staleStartTime;
 338     }
 339
 340     private Object createListener(Class<?> cls) {
 341         try {
 342             Constructor<?> ctor = cls.getConstructor(BgpConfigurationManager.class);
 343             return ctor.newInstance(this);
 344         } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException
 345                 e) {
 346             LOG.error("Failed to create listener object", e);
 347             return null;
 348         }
 349     }
 350
 351     private void registerCallbacks() {
 352         for (Class<?> reactor : REACTORS) {
 353             Object obj = createListener(reactor);
 354             if (obj != null) {
 355                 AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 356                 dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 357                 listeners.add(dcl);
 358             }
 359         }
 360     }
 361
 362     @SuppressWarnings("checkstyle:IllegalCatch")
 363     @PreDestroy
 364     public void close() {
 365         executor.shutdown();
 366
 367         if (bgpThriftService != null) {
 368             bgpThriftService.stop();
 369             bgpThriftService = null;
 370         }
 371
 372         if (isBgpConnected()) {
 373             //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 374             bgpRouter.disconnect();
 375         }
 376
 377         if (candidateRegistration != null) {
 378             candidateRegistration.close();
 379         }
 380
 381         entityListenerRegistration.close();
 382
 383         listeners.forEach(l -> {
 384             try {
 385                 l.close();
 386             } catch (Exception e) {
 387                 LOG.warn("Error closing {}", l ,e);
 388             }
 389         });
 390
 391         LOG.info("{} close", getClass().getSimpleName());
 392     }
 393
 394     private String getProperty(String var, String def) {
 395         String property = bundleContext.getProperty(var);
 396         return property == null ? def : property;
 397     }
 398
 399     private EntityOwnershipCandidateRegistration registerEntityCandidate(
 400             final EntityOwnershipService entityOwnershipService) {
 401         try {
 402             return entityOwnershipService.registerCandidate(
 403                     new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME));
 404         } catch (CandidateAlreadyRegisteredException e) {
 405             LOG.error("failed to register bgp entity", e);
 406             return null;
 407         }
 408     }
 409
 410     private EntityOwnershipListenerRegistration registerEntityListener(
 411             final EntityOwnershipService entityOwnershipService) {
 412         return entityOwnershipService.registerListener(BGP_ENTITY_TYPE_FOR_OWNERSHIP, ownershipChange -> {
 413             LOG.trace("entity owner change event fired: {}", ownershipChange);
 414
 415             if (ownershipChange.getState() == EntityOwnershipChangeState.LOCAL_OWNERSHIP_GRANTED) {
 416                 LOG.trace("This PL is the Owner");
 417                 if (bgpThriftService != null) {
 418                     //opening UPDATE-SERVER port.
 419                     bgpThriftService.start();
 420                 } else {
 421                     LOG.error("I am the owner of BGP entity, but bgpThriftService is not initialized yet");
 422                 }
 423                 bgpRestarted();
 424             } else {
 425                 LOG.debug("Not owner: hasOwner: {}, isOwner: {}", ownershipChange.getState().hasOwner(),
 426                         ownershipChange.getState().isOwner());
 427                 if (bgpThriftService != null && bgpThriftService.isBgpThriftServiceStarted()) {
 428                     //close the bgp Thrift Update-SERVER port opened on non-Entity Owner
 429                     bgpThriftService.stop();
 430                     bgpThriftService = null;
 431                 }
 432                 if (isBgpConnected()) {
 433                     //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 434                     bgpRouter.disconnect();
 435                 }
 436             }
 437         });
 438     }
 439
 440     public boolean isBGPEntityOwner() {
 441         return entityOwnershipUtils.isEntityOwner(new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME), 0, 1);
 442     }
 443
 444     public Bgp get() {
 445         config = getConfig();
 446         return config;
 447     }
 448
 449     public class ConfigServerReactor
 450             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 451             implements ClusteredDataTreeChangeListener<ConfigServer> {
 452         private static final String YANG_OBJ = "config-server ";
 453
 454         public ConfigServerReactor() {
 455             super(ConfigServer.class, ConfigServerReactor.class);
 456         }
 457
 458         @Override
 459         protected void add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 460             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 461             if (!isBGPEntityOwner()) {
 462                 return;
 463             }
 464
 465             try {
 466                 initer.await();
 467             } catch (InterruptedException e) {
 468                 // Ignored
 469             }
 470             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 471             bgpRouter.configServerUpdated();
 472             synchronized (BgpConfigurationManager.this) {
 473                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 474                         val.getPort().intValue());
 475                 if (!res) {
 476                     LOG.error(YANG_OBJ + "Add failed; " + ADD_WARN);
 477                 }
 478             }
 479             VtyshCli.setHostAddr(val.getHost().getValue());
 480             ClearBgpCli.setHostAddr(val.getHost().getValue());
 481         }
 482
 483         @Override
 484         protected ConfigServerReactor getDataTreeChangeListener() {
 485             return ConfigServerReactor.this;
 486         }
 487
 488         @Override
 489         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 490             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 491         }
 492
 493         @Override
 494         protected void remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 495             LOG.trace("received bgp disconnect");
 496             if (!isBGPEntityOwner()) {
 497                 return;
 498             }
 499
 500             bgpRouter.configServerUpdated();
 501
 502             synchronized (BgpConfigurationManager.this) {
 503                 bgpRouter.disconnect();
 504             }
 505         }
 506
 507         @Override
 508         protected void update(InstanceIdentifier<ConfigServer> iid,
 509                 ConfigServer oldval, ConfigServer newval) {
 510             LOG.trace("received bgp Connection update");
 511             if (!isBGPEntityOwner()) {
 512                 return;
 513             }
 514             LOG.error(YANG_OBJ + UPD_WARN);
 515         }
 516     }
 517
 518     private BgpRouter getClient(String yangObj) {
 519         if (bgpRouter == null || !bgpRouter.isBgpConnected()) {
 520             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 521             return null;
 522         }
 523         return bgpRouter;
 524     }
 525
 526     public class AsIdReactor
 527             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 528             implements ClusteredDataTreeChangeListener<AsId> {
 529
 530         private static final String YANG_OBJ = "as-id ";
 531
 532         public AsIdReactor() {
 533             super(AsId.class, AsIdReactor.class);
 534         }
 535
 536         @Override
 537         protected void add(InstanceIdentifier<AsId> iid, AsId val) {
 538             LOG.error("received bgp add asid {}", val);
 539             if (!isBGPEntityOwner()) {
 540                 return;
 541             }
 542             LOG.debug("received add router config asNum {}", val.getLocalAs());
 543             synchronized (BgpConfigurationManager.this) {
 544                 BgpRouter br = getClient(YANG_OBJ);
 545                 if (br == null) {
 546                     LOG.debug("{} Unable to process add for asNum {}; {} {}", YANG_OBJ, val.getLocalAs(),
 547                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 548                     return;
 549                 }
 550                 bgpRestarted();
 551
 552                 startBgpCountersTask();
 553                 startBgpAlarmsTask();
 554             }
 555         }
 556
 557         @Override
 558         protected AsIdReactor getDataTreeChangeListener() {
 559             return AsIdReactor.this;
 560         }
 561
 562         @Override
 563         protected InstanceIdentifier<AsId> getWildCardPath() {
 564             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 565         }
 566
 567         @Override
 568         protected void remove(InstanceIdentifier<AsId> iid, AsId val) {
 569             LOG.error("received delete router config asNum {}", val.getLocalAs());
 570             if (!isBGPEntityOwner()) {
 571                 return;
 572             }
 573             synchronized (BgpConfigurationManager.this) {
 574                 long asNum = val.getLocalAs();
 575                 BgpRouter br = getClient(YANG_OBJ);
 576                 if (br == null) {
 577                     LOG.debug("{} Unable to process remove for asNum {}; {} {}", YANG_OBJ, asNum,
 578                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 579                     return;
 580                 }
 581                 try {
 582                     br.stopBgp(asNum);
 583                 } catch (TException | BgpRouterException e) {
 584                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 585                 }
 586
 587                 stopBgpCountersTask();
 588                 stopBgpAlarmsTask();
 589
 590                 Bgp conf = getConfig();
 591                 if (conf == null) {
 592                     LOG.error("Config Null while removing the as-id");
 593                     return;
 594                 }
 595                 LOG.debug("Removing external routes from FIB");
 596                 deleteExternalFibRoutes();
 597                 List<Neighbors> nbrs = conf.getNeighbors();
 598                 if (nbrs != null && nbrs.size() > 0) {
 599                     LOG.error("Tring to remove the as-id when neighbor config is already present");
 600                     for (Neighbors nbr : nbrs) {
 601                         LOG.debug("Removing Neighbor {} from Data store", nbr.getAddress().getValue());
 602                         delNeighbor(nbr.getAddress().getValue());
 603                     }
 604                 }
 605             }
 606         }
 607
 608         @Override
 609         protected void update(InstanceIdentifier<AsId> iid,
 610                 AsId oldval, AsId newval) {
 611             if (!isBGPEntityOwner()) {
 612                 return;
 613             }
 614             LOG.error(YANG_OBJ + UPD_WARN);
 615         }
 616     }
 617
 618     public class GracefulRestartReactor
 619             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 620             implements ClusteredDataTreeChangeListener<GracefulRestart> {
 621
 622         private static final String YANG_OBJ = "graceful-restart ";
 623
 624         public GracefulRestartReactor() {
 625             super(GracefulRestart.class, GracefulRestartReactor.class);
 626         }
 627
 628         @Override
 629         protected void add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 630             if (!isBGPEntityOwner()) {
 631                 return;
 632             }
 633             synchronized (BgpConfigurationManager.this) {
 634                 int stalePathTime = val.getStalepathTime().intValue();
 635                 BgpRouter br = getClient(YANG_OBJ);
 636                 if (br == null) {
 637                     LOG.error("{} Unable to add stale-path time {}; {} {}", YANG_OBJ, stalePathTime,
 638                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 639                     return;
 640                 }
 641                 try {
 642                     br.addGracefulRestart(stalePathTime);
 643                 } catch (TException | BgpRouterException e) {
 644                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 645                 }
 646             }
 647         }
 648
 649         @Override
 650         protected GracefulRestartReactor getDataTreeChangeListener() {
 651             return GracefulRestartReactor.this;
 652         }
 653
 654         @Override
 655         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 656             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 657         }
 658
 659         @Override
 660         protected void remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 661             if (!isBGPEntityOwner()) {
 662                 return;
 663             }
 664             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 665             synchronized (BgpConfigurationManager.this) {
 666                 BgpRouter br = getClient(YANG_OBJ);
 667                 if (br == null) {
 668                     LOG.error("{} Unable to delete stale-path time; {} {}", YANG_OBJ,
 669                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 670                     return;
 671                 }
 672                 try {
 673                     br.delGracefulRestart();
 674                 } catch (TException | BgpRouterException e) {
 675                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 676                 }
 677             }
 678         }
 679
 680         @Override
 681         protected void update(InstanceIdentifier<GracefulRestart> iid,
 682                 GracefulRestart oldval, GracefulRestart newval) {
 683             if (!isBGPEntityOwner()) {
 684                 return;
 685             }
 686             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 687             synchronized (BgpConfigurationManager.this) {
 688                 int stalePathTime = newval.getStalepathTime().intValue();
 689                 BgpRouter br = getClient(YANG_OBJ);
 690                 if (br == null) {
 691                     LOG.error("{} Unable to update stale-path time to {}; {} {}", YANG_OBJ, stalePathTime,
 692                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 693                     return;
 694                 }
 695                 try {
 696                     br.addGracefulRestart(stalePathTime);
 697                 } catch (TException | BgpRouterException e) {
 698                     LOG.error("{} update received exception; {}", YANG_OBJ, ADD_WARN, e);
 699                 }
 700             }
 701         }
 702     }
 703
 704     public class LoggingReactor
 705             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 706             implements ClusteredDataTreeChangeListener<Logging> {
 707
 708         private static final String YANG_OBJ = "logging ";
 709
 710         public LoggingReactor() {
 711             super(Logging.class, LoggingReactor.class);
 712         }
 713
 714         @Override
 715         protected void add(InstanceIdentifier<Logging> iid, Logging val) {
 716             if (!isBGPEntityOwner()) {
 717                 return;
 718             }
 719             synchronized (BgpConfigurationManager.this) {
 720                 BgpRouter br = getClient(YANG_OBJ);
 721                 if (br == null) {
 722                     LOG.error("{} Unable to add logging for qbgp; {} {}", YANG_OBJ,
 723                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 724                     return;
 725                 }
 726                 try {
 727                     br.setLogging(val.getFile(), val.getLevel());
 728                 } catch (TException | BgpRouterException e) {
 729                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 730                 }
 731             }
 732         }
 733
 734         @Override
 735         protected LoggingReactor getDataTreeChangeListener() {
 736             return LoggingReactor.this;
 737         }
 738
 739         @Override
 740         protected InstanceIdentifier<Logging> getWildCardPath() {
 741             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 742         }
 743
 744         @Override
 745         protected void remove(InstanceIdentifier<Logging> iid, Logging val) {
 746             if (!isBGPEntityOwner()) {
 747                 return;
 748             }
 749             LOG.debug("received remove Logging config val {}", val.getLevel());
 750             synchronized (BgpConfigurationManager.this) {
 751                 BgpRouter br = getClient(YANG_OBJ);
 752                 if (br == null) {
 753                     LOG.error("{} Unable to remove logging for qbgp; {} {}", YANG_OBJ,
 754                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 755                     return;
 756                 }
 757                 try {
 758                     br.setLogging(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
 759                 } catch (TException | BgpRouterException e) {
 760                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 761                 }
 762             }
 763         }
 764
 765         @Override
 766         protected void update(InstanceIdentifier<Logging> iid,
 767                 Logging oldval, Logging newval) {
 768             if (!isBGPEntityOwner()) {
 769                 return;
 770             }
 771             synchronized (BgpConfigurationManager.this) {
 772                 BgpRouter br = getClient(YANG_OBJ);
 773                 if (br == null) {
 774                     LOG.error("{} Unable to update logging for qbgp; {} {}", YANG_OBJ,
 775                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 776                     return;
 777                 }
 778                 try {
 779                     br.setLogging(newval.getFile(), newval.getLevel());
 780                 } catch (TException | BgpRouterException e) {
 781                     LOG.error("{} newval received exception; {}", YANG_OBJ, ADD_WARN, e);
 782                 }
 783             }
 784         }
 785     }
 786
 787     public class NeighborsReactor
 788             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 789             implements ClusteredDataTreeChangeListener<Neighbors> {
 790
 791         private static final String YANG_OBJ = "neighbors ";
 792
 793         public NeighborsReactor() {
 794             super(Neighbors.class, NeighborsReactor.class);
 795         }
 796
 797         @Override
 798         protected void add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 799             if (!isBGPEntityOwner()) {
 800                 return;
 801             }
 802             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 803             synchronized (BgpConfigurationManager.this) {
 804                 String peerIp = val.getAddress().getValue();
 805                 long as = val.getRemoteAs();
 806                 final String md5Secret = extractMd5Secret(val);
 807                 BgpRouter br = getClient(YANG_OBJ);
 808                 if (br == null) {
 809                     LOG.debug("{} Unable to process add for peer {} as {}; {} {}", YANG_OBJ, peerIp, as,
 810                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 811                     return;
 812                 }
 813                 try {
 814                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 815                     br.addNeighbor(peerIp, as, md5Secret);
 816
 817                 } catch (TException | BgpRouterException e) {
 818                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 819                 }
 820             }
 821         }
 822
 823         @Override
 824         protected NeighborsReactor getDataTreeChangeListener() {
 825             return NeighborsReactor.this;
 826         }
 827
 828         @Override
 829         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 830             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class);
 831         }
 832
 833         @Override
 834         protected void remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 835             if (!isBGPEntityOwner()) {
 836                 return;
 837             }
 838             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 839             synchronized (BgpConfigurationManager.this) {
 840                 String peerIp = val.getAddress().getValue();
 841                 BgpRouter br = getClient(YANG_OBJ);
 842                 if (br == null) {
 843                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 844                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 845                     return;
 846                 }
 847                 try {
 848                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 849                     br.delNeighbor(peerIp);
 850                 } catch (TException | BgpRouterException e) {
 851                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 852                 }
 853
 854                 final BgpAlarms bgpAlarms = getBgpAlarms();
 855                 if (bgpAlarms != null) {
 856                     bgpAlarms.clearBgpNbrDownAlarm(peerIp);
 857                 }
 858             }
 859         }
 860
 861         @Override
 862         protected void update(InstanceIdentifier<Neighbors> iid,
 863                 Neighbors oldval, Neighbors newval) {
 864             if (!isBGPEntityOwner()) {
 865                 return;
 866             }
 867             //purposefully nothing to do.
 868         }
 869     }
 870
 871     public class EbgpMultihopReactor
 872             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 873             implements ClusteredDataTreeChangeListener<EbgpMultihop> {
 874
 875         private static final String YANG_OBJ = "ebgp-multihop ";
 876
 877         public EbgpMultihopReactor() {
 878             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 879         }
 880
 881         @Override
 882         protected void add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 883             if (!isBGPEntityOwner()) {
 884                 return;
 885             }
 886             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 887             synchronized (BgpConfigurationManager.this) {
 888                 String peerIp = val.getPeerIp().getValue();
 889                 BgpRouter br = getClient(YANG_OBJ);
 890                 if (br == null) {
 891                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
 892                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 893                     return;
 894                 }
 895                 try {
 896                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 897                 } catch (TException | BgpRouterException e) {
 898                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 899                 }
 900             }
 901         }
 902
 903         @Override
 904         protected EbgpMultihopReactor getDataTreeChangeListener() {
 905             return EbgpMultihopReactor.this;
 906         }
 907
 908         @Override
 909         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
 910             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(EbgpMultihop.class);
 911         }
 912
 913         @Override
 914         protected void remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 915             if (!isBGPEntityOwner()) {
 916                 return;
 917             }
 918             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 919             synchronized (BgpConfigurationManager.this) {
 920                 String peerIp = val.getPeerIp().getValue();
 921                 BgpRouter br = getClient(YANG_OBJ);
 922                 if (br == null) {
 923                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 924                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 925                     return;
 926                 }
 927                 try {
 928                     br.delEbgpMultihop(peerIp);
 929                 } catch (TException | BgpRouterException e) {
 930                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 931                 }
 932             }
 933         }
 934
 935         @Override
 936         protected void update(InstanceIdentifier<EbgpMultihop> iid,
 937                 EbgpMultihop oldval, EbgpMultihop newval) {
 938             if (!isBGPEntityOwner()) {
 939                 return;
 940             }
 941             LOG.error(YANG_OBJ + UPD_WARN);
 942         }
 943     }
 944
 945     public class UpdateSourceReactor
 946             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
 947             implements ClusteredDataTreeChangeListener<UpdateSource> {
 948
 949         private static final String YANG_OBJ = "update-source ";
 950
 951         public UpdateSourceReactor() {
 952             super(UpdateSource.class, UpdateSourceReactor.class);
 953         }
 954
 955         @Override
 956         protected void add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 957             if (!isBGPEntityOwner()) {
 958                 return;
 959             }
 960             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
 961             synchronized (BgpConfigurationManager.this) {
 962                 String peerIp = val.getPeerIp().getValue();
 963                 BgpRouter br = getClient(YANG_OBJ);
 964                 if (br == null) {
 965                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
 966                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 967                     return;
 968                 }
 969                 try {
 970                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
 971                 } catch (TException | BgpRouterException e) {
 972                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 973                 }
 974             }
 975         }
 976
 977         @Override
 978         protected UpdateSourceReactor getDataTreeChangeListener() {
 979             return UpdateSourceReactor.this;
 980         }
 981
 982         @Override
 983         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
 984             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(UpdateSource.class);
 985         }
 986
 987         @Override
 988         protected void remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 989             if (!isBGPEntityOwner()) {
 990                 return;
 991             }
 992             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
 993             synchronized (BgpConfigurationManager.this) {
 994                 String peerIp = val.getPeerIp().getValue();
 995                 BgpRouter br = getClient(YANG_OBJ);
 996                 if (br == null) {
 997                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 998                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 999                     return;
1000                 }
1001                 try {
1002                     br.delUpdateSource(peerIp);
1003                 } catch (TException | BgpRouterException e) {
1004                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1005                 }
1006             }
1007         }
1008
1009         @Override
1010         protected void update(InstanceIdentifier<UpdateSource> iid,
1011                 UpdateSource oldval, UpdateSource newval) {
1012             if (!isBGPEntityOwner()) {
1013                 return;
1014             }
1015             LOG.error(YANG_OBJ + UPD_WARN);
1016         }
1017     }
1018
1019     public class AddressFamiliesReactor
1020             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
1021             implements ClusteredDataTreeChangeListener<AddressFamilies> {
1022
1023         private static final String YANG_OBJ = "address-families ";
1024
1025         public AddressFamiliesReactor() {
1026             super(AddressFamilies.class, AddressFamiliesReactor.class);
1027         }
1028
1029         @Override
1030         protected void add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1031             if (!isBGPEntityOwner()) {
1032                 return;
1033             }
1034             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
1035             synchronized (BgpConfigurationManager.this) {
1036                 String peerIp = val.getPeerIp().getValue();
1037                 BgpRouter br = getClient(YANG_OBJ);
1038                 if (br == null) {
1039                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1040                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1041                     return;
1042                 }
1043                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1044                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1045                 try {
1046                     br.addAddressFamily(peerIp, afi, safi);
1047                 } catch (TException | BgpRouterException e) {
1048                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1049                 }
1050             }
1051         }
1052
1053         @Override
1054         protected AddressFamiliesReactor getDataTreeChangeListener() {
1055             return AddressFamiliesReactor.this;
1056         }
1057
1058         @Override
1059         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
1060             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(AddressFamilies.class);
1061         }
1062
1063         @Override
1064         protected void remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1065             if (!isBGPEntityOwner()) {
1066                 return;
1067             }
1068             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
1069             synchronized (BgpConfigurationManager.this) {
1070                 String peerIp = val.getPeerIp().getValue();
1071                 BgpRouter br = getClient(YANG_OBJ);
1072                 if (br == null) {
1073                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1074                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1075                     return;
1076                 }
1077                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1078                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1079                 try {
1080                     br.delAddressFamily(peerIp, afi, safi);
1081                 } catch (TException | BgpRouterException e) {
1082                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1083                 }
1084             }
1085         }
1086
1087         @Override
1088         protected void update(InstanceIdentifier<AddressFamilies> iid,
1089                 AddressFamilies oldval, AddressFamilies newval) {
1090             if (!isBGPEntityOwner()) {
1091                 return;
1092             }
1093             LOG.error(YANG_OBJ + UPD_WARN);
1094         }
1095     }
1096
1097     public class NetworksReactor
1098             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
1099             implements ClusteredDataTreeChangeListener<Networks> {
1100
1101         private static final String YANG_OBJ = "networks ";
1102
1103         public NetworksReactor() {
1104             super(Networks.class, NetworksReactor.class);
1105         }
1106
1107         @Override
1108         public NetworksReactor getDataTreeChangeListener() {
1109             return NetworksReactor.this;
1110         }
1111
1112         @Override
1113         protected void add(InstanceIdentifier<Networks> iid, Networks val) {
1114             if (!isBGPEntityOwner()) {
1115                 return;
1116             }
1117             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1118             synchronized (BgpConfigurationManager.this) {
1119                 String rd = val.getRd();
1120                 String pfxlen = val.getPrefixLen();
1121                 String nh = val.getNexthop().getValue();
1122                 BgpRouter br = getClient(YANG_OBJ);
1123                 if (br == null) {
1124                     LOG.debug("{} Unable to process add for rd {} prefix {} nexthop {}; {} {}", YANG_OBJ, rd, pfxlen,
1125                             nh, BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1126                     return;
1127                 }
1128                 Long label = val.getLabel();
1129                 int lbl = label == null ? qbgpConstants.LBL_NO_LABEL
1130                         : label.intValue();
1131                 int l3vni = val.getL3vni() == null ? qbgpConstants.LBL_NO_LABEL
1132                         : val.getL3vni().intValue();
1133                 int l2vni = val.getL2vni() == null ? qbgpConstants.LBL_NO_LABEL
1134                         : val.getL2vni().intValue();
1135
1136                 BgpControlPlaneType protocolType = val.getBgpControlPlaneType();
1137                 int ethernetTag = val.getEthtag().intValue();
1138                 String esi = val.getEsi();
1139                 String macaddress = val.getMacaddress();
1140                 EncapType encapType = val.getEncapType();
1141                 String routerMac = val.getRoutermac();
1142
1143                 try {
1144                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni, BgpUtil.convertToThriftProtocolType(protocolType),
1145                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1146                 } catch (TException | BgpRouterException e) {
1147                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1148                 }
1149             }
1150         }
1151
1152         @Override
1153         protected InstanceIdentifier<Networks> getWildCardPath() {
1154             return InstanceIdentifier.create(Bgp.class).child(Networks.class);
1155         }
1156
1157         @Override
1158         protected void remove(InstanceIdentifier<Networks> iid, Networks val) {
1159             if (!isBGPEntityOwner()) {
1160                 return;
1161             }
1162             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1163             synchronized (BgpConfigurationManager.this) {
1164                 String rd = val.getRd();
1165                 String pfxlen = val.getPrefixLen();
1166                 BgpRouter br = getClient(YANG_OBJ);
1167                 if (br == null) {
1168                     LOG.debug("{} Unable to process remove for rd {} prefix {}; {} {}", YANG_OBJ, rd, pfxlen,
1169                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1170                     return;
1171                 }
1172                 Long label = val.getLabel();
1173                 int lbl = label == null ? 0 : label.intValue();
1174                 if (rd == null && lbl > 0) {
1175                     //LU prefix is being deleted.
1176                     rd = Integer.toString(lbl);
1177                 }
1178                 try {
1179                     br.delPrefix(rd, pfxlen);
1180                 } catch (TException | BgpRouterException e) {
1181                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1182                 }
1183             }
1184         }
1185
1186         /**get the value AFI from a prefix as "x.x.x.x/x".
1187          *
1188          * @param pfxlen the prefix to get an afi
1189          * @return the afi value as you are need
1190          */
1191         public  int  testValueAFI(String pfxlen) {
1192             int afiNew = af_afi.AFI_IP.getValue();
1193             try {
1194                 String ipOnly = pfxlen.substring(0, pfxlen.lastIndexOf("/"));
1195                 java.net.Inet6Address.getByName(ipOnly);
1196                 afiNew = af_afi.AFI_IPV6.getValue();
1197             } catch (java.net.UnknownHostException e) {
1198                 //ce n'est pas de l'ipv6
1199             }
1200             return afiNew;
1201         }
1202
1203
1204         @Override
1205         protected void update(final InstanceIdentifier<Networks> iid,
1206                 final Networks oldval, final Networks newval) {
1207             if (!isBGPEntityOwner()) {
1208                 return;
1209             }
1210             if (oldval.equals(newval)) {
1211                 //Update: OLD and New values are same, no need to trigger remove/add.
1212                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1213                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1214                 return;
1215             }
1216             LOG.debug("update networks old val RD: {}, Prefix: {}, Label: {}, NH: {} "
1217                             + "new val RD: {}, Prefix: {}, Label: {}, NH: {}",
1218                     oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop(),
1219                     newval.getRd(), newval.getPrefixLen(), newval.getLabel(), newval.getNexthop());
1220             remove(iid, oldval);
1221             add(iid, newval);
1222         }
1223     }
1224
1225     static Timer timer = new Timer();
1226
1227     public class VrfsReactor
1228             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1229             implements ClusteredDataTreeChangeListener<Vrfs> {
1230
1231         private static final String YANG_OBJ = "vrfs ";
1232
1233         public VrfsReactor() {
1234             super(Vrfs.class, VrfsReactor.class);
1235         }
1236
1237         @Override
1238         protected void add(InstanceIdentifier<Vrfs> iid, Vrfs vrfs) {
1239             if (!isBGPEntityOwner()) {
1240                 return;
1241             }
1242             LOG.debug("received add Vrfs config value {}", vrfs.getRd());
1243             synchronized (BgpConfigurationManager.this) {
1244                 String rd = vrfs.getRd();
1245                 BgpRouter br = getClient(YANG_OBJ);
1246                 if (br == null) {
1247                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1248                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1249                     return;
1250                 }
1251                 try {
1252                     List<AddressFamiliesVrf> vrfAddrFamilyList = vrfs.getAddressFamiliesVrf();
1253                     for (AddressFamiliesVrf vrfAddrFamily : vrfAddrFamilyList) {
1254                         /*add to br the new vrfs arguments*/
1255                         br.addVrf(BgpUtil.getLayerType(vrfAddrFamily), rd, vrfs.getImportRts(), vrfs.getExportRts()
1256                         );
1257                     }
1258                     /*add to br the vrfs contained in mapNewAdFamily*/
1259                     List<AddressFamiliesVrf> vrfAddrFamilyListFromMap = mapNewAdFamily.get(rd);
1260                     if (vrfAddrFamilyListFromMap == null) {
1261                         return;
1262                     }
1263
1264                     for (AddressFamiliesVrf adf : vrfAddrFamilyListFromMap) {
1265                         if (vrfAddrFamilyList.contains(adf)) {
1266                             mapNewAdFamily.remove(rd);
1267                         } else  if (adf != null) {
1268
1269                             br.addVrf(BgpUtil.getLayerType(adf), rd, vrfs.getImportRts(), vrfs.getExportRts()
1270                             );
1271                             // remove AddressFamiliesVrf which was already added to BGP
1272                             vrfAddrFamilyListFromMap.remove(adf);
1273                             if (vrfAddrFamilyListFromMap.isEmpty()) {
1274                                 // remove Vrf entry from temp mapNewAdFamily if all its AddressFamiliesVrf was
1275                                 // added to BGP
1276                                 mapNewAdFamily.remove(rd);
1277                             }
1278                         }
1279                     }
1280                 } catch (TException | BgpRouterException e) {
1281                     LOG.error("{} get {}, Add received exception", YANG_OBJ, ADD_WARN, e);
1282                 }
1283             }
1284         }
1285
1286         @Override
1287         protected VrfsReactor getDataTreeChangeListener() {
1288             return VrfsReactor.this;
1289         }
1290
1291         @Override
1292         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1293             return InstanceIdentifier.create(Bgp.class).child(Vrfs.class);
1294         }
1295
1296         @Override
1297         protected void remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1298             if (!isBGPEntityOwner()) {
1299                 return;
1300             }
1301             LOG.debug("received remove Vrfs config val {}", val.getRd());
1302             synchronized (BgpConfigurationManager.this) {
1303                 String rd = val.getRd();
1304                 BgpRouter br = getClient(YANG_OBJ);
1305                 if (br == null) {
1306                     LOG.debug("{} Unable to process remove for rd {}; {} {}", YANG_OBJ, rd,
1307                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1308                     return;
1309                 }
1310                 try {
1311                     List<AddressFamiliesVrf> adf = mapNewAdFamily.get(rd);
1312                     adf = adf != null ? adf : new ArrayList<>();
1313                     for (AddressFamiliesVrf s : val.getAddressFamiliesVrf()) {
1314                         br.delVrf(rd, s.getAfi(), s.getSafi());
1315                         adf.remove(s);// remove in the map the vrf in waiting for advertise quagga
1316                     }
1317                     if (adf.isEmpty()) {
1318                         mapNewAdFamily.remove(rd);
1319                     }
1320                 } catch (TException | BgpRouterException e) {
1321                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1322                 }
1323             }
1324         }
1325
1326         @Override
1327         protected void update(InstanceIdentifier<Vrfs> iid,
1328                 Vrfs oldval, Vrfs newval) {
1329             if (oldval != null && newval != null) {
1330                 LOG.debug("received update Vrfs config val {}, VRFS: Update getting triggered for VRFS rd {}",
1331                         newval.getRd(), oldval.getRd());
1332             } else {
1333                 LOG.debug("received update Vrfs config val {}, from old vrf {}",
1334                         newval, oldval);
1335             }
1336             if (!isBGPEntityOwner()) {
1337                 return;
1338             }
1339
1340             List<AddressFamiliesVrf> adFamilyVrfToDel = new ArrayList<>();
1341             List<AddressFamiliesVrf> adFamilyVrfToAdd = new ArrayList<>();
1342             List<AddressFamiliesVrf> oldlistAdFamilies = new ArrayList<>();
1343             List<AddressFamiliesVrf> newlistAdFamilies = new ArrayList<>();
1344             if (oldval != null) {
1345                 oldlistAdFamilies = oldval.getAddressFamiliesVrf() == null
1346                         ? new ArrayList<>() : oldval.getAddressFamiliesVrf();
1347             }
1348             if (newval != null) {
1349                 newlistAdFamilies = newval.getAddressFamiliesVrf() == null
1350                         ? new ArrayList<>() : newval.getAddressFamiliesVrf();
1351             }
1352             /*find old AddressFamily to remove from new configuration*/
1353             for (AddressFamiliesVrf adVrf : oldlistAdFamilies) {
1354                 if (!newlistAdFamilies.contains(adVrf)) {
1355                     adFamilyVrfToDel.add(adVrf);
1356                 }
1357             }
1358             /*find new AddressFamily to add to unexisting configuration*/
1359             for (AddressFamiliesVrf adVrf : newlistAdFamilies) {
1360                 if (!oldlistAdFamilies.contains(adVrf)) {
1361                     adFamilyVrfToAdd.add(adVrf);
1362                 }
1363             }
1364             String rd = newval != null ? newval.getRd() : null;
1365             if (rd != null) {
1366                 BgpRouter br = getClient(YANG_OBJ);
1367                 if (br == null) {
1368                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1369                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1370                     return;
1371                 }
1372
1373                 for (AddressFamiliesVrf adfvrf : adFamilyVrfToAdd) {
1374                     try {
1375                         LOG.debug("call addVRf rd {} afi {} safi {}", rd, adfvrf.getAfi(), adfvrf.getSafi());
1376                         br.addVrf(BgpUtil.getLayerType(adfvrf), rd, newval.getImportRts(),
1377                                 newval.getExportRts());
1378                     } catch (TException | BgpRouterException e) {
1379                         LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1380                     }
1381                 }
1382
1383                 for (AddressFamiliesVrf adfToDel : adFamilyVrfToDel) {
1384                     try {
1385                         LOG.debug("call delVRf rd {} afi {} safi {}", rd, adfToDel.getAfi(), adfToDel.getSafi());
1386                         br.delVrf(rd, adfToDel.getAfi(), adfToDel.getSafi());
1387                     } catch (TException | BgpRouterException e) {
1388                         LOG.error("{} delVrf received exception; {}", YANG_OBJ, ADD_WARN, e);
1389                     }
1390                 }
1391             }
1392         }
1393     }
1394
1395     protected void activateMIP() {
1396         try {
1397             LOG.trace("BgpReactor: Executing MIP Activate command");
1398             Runtime.getRuntime().exec("cluster ip -a sdnc_bgp_mip");
1399             Runtime.getRuntime().exec("cluster ip -a sdnc_os_mip");
1400             LOG.trace("bgpMIP Activated");
1401
1402         } catch (IOException io) {
1403             LOG.error("IO Exception got while activating mip: {}", io.getMessage());
1404         }
1405     }
1406
1407     public class BgpReactor
1408             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1409             implements ClusteredDataTreeChangeListener<Bgp> {
1410
1411         private static final String YANG_OBJ = "Bgp ";
1412
1413         public BgpReactor() {
1414             super(Bgp.class, BgpReactor.class);
1415         }
1416
1417
1418         @Override
1419         protected void add(InstanceIdentifier<Bgp> iid, Bgp val) {
1420             LOG.debug("received add Bgp config");
1421
1422             try {
1423                 initer.await();
1424             } catch (InterruptedException e) {
1425                 // Ignored
1426             }
1427             synchronized (BgpConfigurationManager.this) {
1428                 config = val;
1429                 if (!isBGPEntityOwner()) {
1430                     return;
1431                 }
1432                 activateMIP();
1433             }
1434         }
1435
1436         @Override
1437         protected BgpReactor getDataTreeChangeListener() {
1438             return BgpReactor.this;
1439         }
1440
1441         @Override
1442         protected InstanceIdentifier<Bgp> getWildCardPath() {
1443             return InstanceIdentifier.create(Bgp.class);
1444         }
1445
1446         @Override
1447         protected void remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1448             if (!isBGPEntityOwner()) {
1449                 return;
1450             }
1451             LOG.debug("received remove Bgp config");
1452
1453             config = null;
1454         }
1455
1456         @Override
1457         protected void update(InstanceIdentifier<Bgp> iid,
1458                 Bgp oldval, Bgp newval) {
1459             if (!isBGPEntityOwner()) {
1460                 return;
1461             }
1462
1463             config = newval;
1464         }
1465     }
1466
1467     @SuppressWarnings("deprecation")
1468     public class MultipathReactor
1469             extends AsyncDataTreeChangeListenerBase<Multipath, MultipathReactor>
1470             implements ClusteredDataTreeChangeListener<Multipath> {
1471
1472         private static final String YANG_OBJ = "multipath ";
1473
1474         public MultipathReactor() {
1475             super(Multipath.class, MultipathReactor.class);
1476         }
1477
1478
1479         @Override
1480         protected MultipathReactor getDataTreeChangeListener() {
1481             return MultipathReactor.this;
1482         }
1483
1484         @Override
1485         protected InstanceIdentifier<Multipath> getWildCardPath() {
1486             return InstanceIdentifier.create(Bgp.class).child(Multipath.class);
1487         }
1488
1489         @Override
1490         protected void remove(InstanceIdentifier<Multipath> iid, Multipath val) {
1491             executor.execute(new MultipathStatusChange(val));
1492         }
1493
1494         @Override
1495         protected void update(InstanceIdentifier<Multipath> iid, Multipath oldval, Multipath newval) {
1496             executor.execute(new MultipathStatusChange(newval));
1497         }
1498
1499         @Override
1500         protected void add(InstanceIdentifier<Multipath> key, Multipath dataObjectModification) {
1501             executor.execute(new MultipathStatusChange(dataObjectModification));
1502         }
1503
1504         class MultipathStatusChange implements Runnable {
1505
1506             Multipath multipath;
1507
1508             MultipathStatusChange(Multipath multipath) {
1509                 this.multipath = multipath;
1510             }
1511
1512             @Override
1513             public void run() {
1514                 if (isBGPEntityOwner()) {
1515                     synchronized (BgpConfigurationManager.this) {
1516
1517                         BgpRouter br = getClient(YANG_OBJ);
1518
1519                         if (br != null) {
1520                             af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1521                             af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1522
1523                             try {
1524                                 if (multipath.isMultipathEnabled()) {
1525                                     br.enableMultipath(afi, safi);
1526                                 } else {
1527                                     br.disableMultipath(afi, safi);
1528                                 }
1529                             } catch (TException | BgpRouterException e) {
1530                                 LOG.error("{} received exception", YANG_OBJ, e);
1531                             }
1532                         }
1533                     }
1534                 }
1535             }
1536
1537         }
1538
1539         @Override
1540         public void close() {
1541             super.close();
1542         }
1543     }
1544
1545     @SuppressWarnings("deprecation")
1546     public class VrfMaxpathReactor
1547             extends AsyncDataTreeChangeListenerBase<VrfMaxpath, VrfMaxpathReactor>
1548             implements ClusteredDataTreeChangeListener<VrfMaxpath> {
1549
1550         private static final String YANG_OBJ = "vrfMaxpath ";
1551
1552         public VrfMaxpathReactor() {
1553             super(VrfMaxpath.class, VrfMaxpathReactor.class);
1554         }
1555
1556
1557         @Override
1558         protected VrfMaxpathReactor getDataTreeChangeListener() {
1559             return VrfMaxpathReactor.this;
1560         }
1561
1562         @Override
1563         protected InstanceIdentifier<VrfMaxpath> getWildCardPath() {
1564             return InstanceIdentifier.create(Bgp.class).child(VrfMaxpath.class);
1565         }
1566
1567         class VrfMaxPathConfigurator implements Runnable {
1568
1569             VrfMaxpath vrfMaxpathVal;
1570
1571             VrfMaxPathConfigurator(VrfMaxpath vrfMaxPathVal) {
1572                 this.vrfMaxpathVal = vrfMaxPathVal;
1573             }
1574
1575             @Override
1576             public void run() {
1577                 if (isBGPEntityOwner()) {
1578                     synchronized (BgpConfigurationManager.this) {
1579                         BgpRouter br = getClient(YANG_OBJ);
1580                         if (br != null) {
1581                             try {
1582                                 br.multipaths(vrfMaxpathVal.getRd(), vrfMaxpathVal.getMaxpaths());
1583                                 LOG.debug("Maxpath for vrf {} is {}", vrfMaxpathVal.getRd(),
1584                                         vrfMaxpathVal.getMaxpaths());
1585                             } catch (TException | BgpRouterException e) {
1586                                 LOG.error("{} received exception", YANG_OBJ, e);
1587                             }
1588                         }
1589                     }
1590                 }
1591             }
1592         }
1593
1594         @Override
1595         protected void remove(InstanceIdentifier<VrfMaxpath> iid, VrfMaxpath vrfMaxPathVal) {
1596             executor.execute(new VrfMaxPathConfigurator(vrfMaxPathVal));
1597         }
1598
1599         @Override
1600         protected void update(InstanceIdentifier<VrfMaxpath> iid,
1601                               VrfMaxpath oldval, VrfMaxpath newval) {
1602             if (!Objects.equals(oldval.getMaxpaths(), newval.getMaxpaths())) {
1603                 executor.execute(new VrfMaxPathConfigurator(newval));
1604             }
1605         }
1606
1607         @Override
1608         protected void add(InstanceIdentifier<VrfMaxpath> instanceIdentifier, VrfMaxpath vrfMaxpathVal) {
1609             executor.execute(new VrfMaxPathConfigurator(vrfMaxpathVal));
1610         }
1611
1612         @Override
1613         public void close() {
1614             super.close();
1615         }
1616     }
1617
1618     public boolean isIpAvailable(String odlip) {
1619
1620         try {
1621             if (odlip != null) {
1622                 if ("127.0.0.1".equals(odlip)) {
1623                     return true;
1624                 }
1625                 Enumeration<NetworkInterface> networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces();
1626                 while (networkInterfaceEnumeration.hasMoreElements()) {
1627                     NetworkInterface networkInterface = networkInterfaceEnumeration.nextElement();
1628                     Enumeration<InetAddress>  inetAddressEnumeration = networkInterface.getInetAddresses();
1629                     while (inetAddressEnumeration.hasMoreElements()) {
1630                         InetAddress inetAddress = inetAddressEnumeration.nextElement();
1631                         if (odlip.equals(inetAddress.getHostAddress())) {
1632                             return true;
1633                         }
1634                     }
1635                 }
1636             }
1637         } catch (SocketException e) {
1638             // Ignored?
1639         }
1640         return false;
1641     }
1642
1643     public long getStalePathtime(int defValue, AsId asId) {
1644         long spt = 0;
1645         try {
1646             spt = getConfig().getGracefulRestart().getStalepathTime();
1647         } catch (NullPointerException e) {
1648             try {
1649                 spt = asId.getStalepathTime();
1650                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1651             } catch (NullPointerException ignore) {
1652                 LOG.trace("BGP AS id is not set using graceful");
1653                 spt = defValue;
1654             }
1655         }
1656         if (spt == 0) {
1657             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1658             spt = defValue;
1659         }
1660         return spt;
1661     }
1662
1663     public static boolean isValidConfigBgpHostPort(String bgpHost, int bgpPort) {
1664         if (!bgpHost.equals(DEF_CHOST)) {
1665             return true;
1666         } else {
1667             return false;
1668         }
1669     }
1670
1671     public synchronized void bgpRestarted() {
1672         /*
1673          * If there a thread which in the process of stale cleanup, cancel it
1674          * and start a new thread (to avoid processing same again).
1675          */
1676         if (previousReplayJobInProgress()) {
1677             cancelPreviousReplayJob();
1678         }
1679         Runnable task = () -> {
1680             LOG.info("running bgp replay task ");
1681             if (get() == null) {
1682                 String host = getConfigHost();
1683                 int port = getConfigPort();
1684                 LOG.info("connecting  to bgp host {} ", host);
1685                 bgpRouter.connect(host, port);
1686                 LOG.info("no config to push in bgp replay task ");
1687                 return;
1688             }
1689             setStaleStartTime(System.currentTimeMillis());
1690             LOG.info("started creating stale fibDSWriter  map ");
1691             createStaleFibMap();
1692             setStaleEndTime(System.currentTimeMillis());
1693             LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
1694             LOG.info("started bgp config replay ");
1695             setCfgReplayStartTime(System.currentTimeMillis());
1696             boolean replaySucceded = replay();
1697             setCfgReplayEndTime(System.currentTimeMillis());
1698             LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1699             if (replaySucceded) {
1700                 LOG.info("starting the stale cleanup timer");
1701                 long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1702                 setStaleCleanupTime(routeSyncTime);
1703                 routeCleanupFuture = executor.schedule(new RouteCleanup(), routeSyncTime, TimeUnit.SECONDS);
1704             } else {
1705                 staledFibEntriesMap.clear();
1706             }
1707         };
1708         lastReplayJobFt = executor.submit(task);
1709     }
1710
1711     private boolean previousReplayJobInProgress() {
1712         return lastReplayJobFt != null && !lastReplayJobFt.isDone();
1713     }
1714
1715     private void cancelPreviousReplayJob() {
1716         try {
1717             LOG.error("cancelling already running bgp replay task");
1718             if (lastReplayJobFt != null) {
1719                 lastReplayJobFt.cancel(true);
1720                 lastReplayJobFt = null;
1721                 staledFibEntriesMap.clear();
1722             }
1723             if (routeCleanupFuture != null) {
1724                 routeCleanupFuture.cancel(true);
1725                 routeCleanupFuture = null;
1726                 staledFibEntriesMap.clear();
1727             }
1728             Thread.sleep(2000);
1729         } catch (InterruptedException e) {
1730             LOG.error("Failed to cancel previous replay job ", e);
1731         }
1732     }
1733
1734     private void doRouteSync() {
1735         LOG.error("Starting BGP route sync");
1736         try {
1737             bgpRouter.initRibSync(bgpSyncHandle);
1738         } catch (BgpRouterException e) {
1739             LOG.error("Route sync aborted, exception when initializing", e);
1740             return;
1741         }
1742         while (bgpSyncHandle.getState() != BgpSyncHandle.DONE) {
1743             for (af_afi afi : af_afi.values()) {
1744                 Routes routes = null;
1745                 try {
1746                     routes = bgpRouter.doRibSync(bgpSyncHandle, afi);
1747                 } catch (TException | BgpRouterException e) {
1748                     LOG.error("Route sync aborted, exception when syncing", e);
1749                     return;
1750                 }
1751                 Iterator<Update> updates = routes.getUpdatesIterator();
1752                 while (updates.hasNext()) {
1753                     Update update = updates.next();
1754                     String rd = update.getRd();
1755                     String nexthop = update.getNexthop();
1756
1757                     // TODO: decide correct label here
1758                     int label = update.getL3label();
1759                     int l2label = update.getL2label();
1760
1761                     String prefix = update.getPrefix();
1762                     int plen = update.getPrefixlen();
1763
1764
1765                     // TODO: protocol type will not be available in "update"
1766                     // use "rd" to query vrf table and obtain the protocol_type.
1767                     // Currently using PROTOCOL_EVPN as default.
1768                     onUpdatePushRoute(
1769                            protocol_type.PROTOCOL_EVPN,
1770                            rd,
1771                            prefix,
1772                            plen,
1773                            nexthop,
1774                            update.getMacaddress(),
1775                            label,
1776                            l2label,
1777                            update.getRoutermac(),
1778                            afi);
1779                 }
1780             }
1781         }
1782         try {
1783             LOG.error("Ending BGP route-sync");
1784             bgpRouter.endRibSync(bgpSyncHandle);
1785         } catch (BgpRouterException e) {
1786             // Ignored?
1787         }
1788     }
1789
1790     public void addTepToElanDS(String rd, String tepIp, String mac, Long l2vni) {
1791         boolean needUpdate = addToRt2TepMap(rd, tepIp, mac, l2vni);
1792         if (needUpdate) {
1793             LOG.info("Adding tepIp {} with RD {} to ELan DS", tepIp, rd);
1794             bgpUtil.addTepToElanInstance(rd, tepIp);
1795         } else {
1796             LOG.debug("Skipping the Elan update for RT2 from tep {} rd {}", tepIp, rd);
1797         }
1798     }
1799
1800     public void deleteTepfromElanDS(String rd, String tepIp, String mac) {
1801         boolean needUpdate = deleteFromRt2TepMap(rd, tepIp, mac);
1802         if (needUpdate) {
1803             LOG.info("Deleting tepIp {} with RD {} to ELan DS", tepIp, rd);
1804             bgpUtil.deleteTepFromElanInstance(rd, tepIp);
1805         } else {
1806             LOG.debug("Skipping the Elan update for RT2 withdraw from tep {} rd {}", tepIp, rd);
1807         }
1808     }
1809
1810     /* onUpdatePushRoute
1811      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1812      *  - Entry compare shall include NextHop, Label.
1813      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1814      *  - If entry not found, add to FIB Config DS.
1815      *  - If entry found, but either Label/NextHop doesn't match.
1816      *      - Update FIB Config DS with modified values.
1817      *      - delete from Stale Map.
1818      */
1819
1820     public void onUpdatePushRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
1821                                   String macaddress, int label, int l2label, String routermac, af_afi afi) {
1822         boolean addroute = false;
1823         boolean macupdate = false;
1824         long l3vni = 0L;
1825         VrfEntry.EncapType encapType = VrfEntry.EncapType.Mplsgre;
1826         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1827             encapType = VrfEntry.EncapType.Vxlan;
1828             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
1829             if (vpnInstanceOpDataEntry != null) {
1830                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
1831                     LOG.info("Got RT2 route for RD {} l3label {} l2label {} from tep {} with mac {} remote RD {}",
1832                             vpnInstanceOpDataEntry.getVpnInstanceName(), label, l2label, nextHop, macaddress, rd);
1833                     addTepToElanDS(rd, nextHop, macaddress, (long)l2label);
1834                     macupdate = true;
1835                 } else {
1836                     l3vni = vpnInstanceOpDataEntry.getL3vni();
1837                 }
1838             } else {
1839                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1840                 return;
1841             }
1842         }
1843
1844         if (!staledFibEntriesMap.isEmpty()) {
1845             // restart Scenario, as MAP is not empty.
1846             Map<String, Long> map = staledFibEntriesMap.get(rd);
1847             if (map != null) {
1848                 String prefixNextHop = appendNextHopToPrefix(prefix + "/" + plen, nextHop);
1849                 Long labelInStaleMap = map.get(prefixNextHop);
1850                 if (null == labelInStaleMap) {
1851                     // New Entry, which happened to be added during restart.
1852                     addroute = true;
1853                 } else {
1854                     map.remove(prefixNextHop);
1855                     if (isRouteModified(label, labelInStaleMap)) {
1856                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1857                         // Existing entry, where in Label got modified during restart
1858                         addroute = true;
1859                     }
1860                 }
1861             } else {
1862                 LOG.debug("rd {} map is null while processing prefix {} ", rd, prefix);
1863                 addroute = true;
1864             }
1865         } else {
1866             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1867             addroute = true;
1868         }
1869         if (macupdate) {
1870             LOG.info("ADD: Adding Mac Fib entry rd {} mac{} nexthop {} l2vni {}", rd, macaddress, nextHop, l2label);
1871             fibDSWriter.addMacEntryToDS(rd, macaddress, prefix, Collections.singletonList(nextHop),
1872                     encapType, l2label, routermac, RouteOrigin.BGP);
1873             LOG.info("ADD: Added Mac Fib entry rd {} prefix {} nexthop {} label {}", rd, macaddress, nextHop, l2label);
1874         } else if (addroute) {
1875             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {} afi {}",
1876                     rd, prefix, nextHop, label, afi);
1877             // TODO: modify addFibEntryToDS signature
1878             List<String> nextHopList = Collections.singletonList(nextHop);
1879             fibDSWriter.addFibEntryToDS(rd, prefix + "/" + plen, nextHopList, encapType, label, l3vni,
1880                                         routermac, RouteOrigin.BGP);
1881             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1882             String vpnName = bgpUtil.getVpnNameFromRd(rd);
1883             if (vpnName != null) {
1884                 vpnLinkService.leakRouteIfNeeded(vpnName, prefix, nextHopList, label, RouteOrigin.BGP,
1885                                                  NwConstants.ADD_FLOW);
1886             }
1887         }
1888     }
1889
1890     public void onUpdateWithdrawRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
1891             String macaddress) {
1892         long vni = 0L;
1893         boolean macupdate = false;
1894         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1895             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
1896             if (vpnInstanceOpDataEntry != null) {
1897                 vni = vpnInstanceOpDataEntry.getL3vni();
1898                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
1899                     LOG.debug("Got RT2 withdraw for RD {} {} from tep {} with mac {} remote RD {}",
1900                             vpnInstanceOpDataEntry.getVpnInstanceName(), vni, nextHop, macaddress, rd);
1901                     deleteTepfromElanDS(rd, nextHop, macaddress);
1902                     LOG.debug("For rd {}. skipping fib update", rd);
1903                     macupdate = true;
1904                 }
1905             } else {
1906                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1907                 return;
1908             }
1909         }
1910         if (macupdate) {
1911             LOG.info("Removing Mac Fib entry rd {} mac{} nexthop {} ", rd, macaddress, nextHop);
1912             fibDSWriter.removeMacEntryFromDS(rd, macaddress);
1913             LOG.info("Removed Mac Fib entry rd {} prefix {} nexthop {} ", rd, macaddress, nextHop);
1914         } else {
1915             LOG.info("REMOVE: Removing Fib entry rd {} prefix {}", rd, prefix);
1916             fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nextHop);
1917             LOG.info("REMOVE: Removed Fib entry rd {} prefix {}", rd, prefix);
1918         }
1919     }
1920
1921     //TODO: below function is for testing purpose with cli
1922     public void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
1923         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
1924         fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nexthop);
1925         String vpnName = bgpUtil.getVpnNameFromRd(rd);
1926         if (vpnName != null) {
1927             vpnLinkService.leakRouteIfNeeded(vpnName, prefix, null /*nextHopList*/, 0 /*INVALID_LABEL*/,
1928                                              RouteOrigin.BGP, NwConstants.DEL_FLOW);
1929         }
1930     }
1931
1932     private static boolean isRouteModified(int label, Long labelInStaleMap) {
1933         return labelInStaleMap != null && !labelInStaleMap.equals(Long.valueOf(label));
1934     }
1935
1936     static class ReplayNbr {
1937         Neighbors nbr;
1938         boolean shouldRetry = false;
1939
1940         public Neighbors getNbr() {
1941             return nbr;
1942         }
1943
1944         public boolean isShouldRetry() {
1945             return shouldRetry;
1946         }
1947
1948         public void setShouldRetry(boolean retryNbr) {
1949             this.shouldRetry = retryNbr;
1950         }
1951
1952         ReplayNbr(Neighbors nbr, boolean shouldRetry) {
1953             this.nbr = nbr;
1954             this.shouldRetry = shouldRetry;
1955         }
1956     }
1957
1958     private static boolean replayNbrConfig(List<Neighbors> neighbors, BgpRouter br) {
1959         if (neighbors == null || neighbors.isEmpty()) {
1960             LOG.error("Replaying nbr configuration, received NULL list ");
1961             return true;
1962         }
1963
1964         List<ReplayNbr> replayNbrList = new ArrayList<>();
1965         for (Neighbors nbr : neighbors) {
1966             if (nbr != null) {
1967                 replayNbrList.add(new ReplayNbr(nbr, true));
1968             }
1969         }
1970         final int numberOfNbrRetries = 3;
1971         RetryOnException nbrRetry = new RetryOnException(numberOfNbrRetries);
1972         do {
1973             for (ReplayNbr replayNbr : replayNbrList) {
1974                 if (!replayNbr.isShouldRetry()) {
1975                     continue;
1976                 }
1977                 boolean replayDone = false;
1978                 LOG.debug("Replaying addNbr {}", replayNbr.getNbr().getAddress().getValue());
1979                 replayDone = false;
1980                 try {
1981                     final String md5password = extractMd5Secret(replayNbr.getNbr());
1982                     br.addNeighbor(replayNbr.getNbr().getAddress().getValue(),
1983                             replayNbr.getNbr().getRemoteAs().longValue(), md5password);
1984                     UpdateSource us = replayNbr.getNbr().getUpdateSource();
1985                     if (us != null) {
1986                         LOG.debug("Replaying updatesource along with nbr: {} US-ip: {} to peer {}",
1987                                 replayNbr.getNbr().getAddress().getValue(),
1988                                 us.getSourceIp().getValue(),
1989                                 us.getPeerIp().getValue());
1990                         br.addUpdateSource(us.getPeerIp().getValue(),
1991                                 us.getSourceIp().getValue());
1992                     }
1993                     replayDone = true;
1994                 } catch (TException | BgpRouterException eNbr) {
1995                     LOG.debug("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), eNbr);
1996                 }
1997                 boolean replaySuccess = true;
1998                 replaySuccess = replaySuccess && replayDone;
1999                 LOG.debug("Replay addNbr {} successful", replayNbr.getNbr().getAddress().getValue());
2000
2001                 //Update Source handling
2002                 UpdateSource us = replayNbr.getNbr().getUpdateSource();
2003                 if (replayDone == false && us != null) {
2004                     LOG.debug("Replaying updatesource {} to peer {}", us.getSourceIp().getValue(),
2005                             us.getPeerIp().getValue());
2006                     replayDone = false;
2007                     try {
2008                         br.addUpdateSource(us.getPeerIp().getValue(),
2009                                 us.getSourceIp().getValue());
2010                         replayDone = true;
2011                     } catch (TException | BgpRouterException eUs) {
2012                         LOG.debug("Replaying UpdateSource for Nbr {}, exception:",
2013                                 replayNbr.getNbr().getAddress().getValue(), eUs);
2014                     }
2015                     LOG.debug("Replay updatesource {} successful", us.getSourceIp().getValue());
2016                     replaySuccess = replaySuccess && replayDone;
2017                 }
2018                 //Ebgp Multihope
2019                 EbgpMultihop en = replayNbr.getNbr().getEbgpMultihop();
2020                 if (en != null) {
2021                     replayDone = false;
2022                     try {
2023                         br.addEbgpMultihop(en.getPeerIp().getValue(),
2024                                 en.getNhops().intValue());
2025                         replayDone = true;
2026                     } catch (TException | BgpRouterException eEbgpMhop) {
2027                         LOG.debug("Replaying EbgpMultihop for Nbr {}, exception: ",
2028                                 replayNbr.getNbr().getAddress().getValue(), eEbgpMhop);
2029                     }
2030                     replaySuccess = replaySuccess && replayDone;
2031                 }
2032
2033                 //afs
2034                 List<AddressFamilies> afs = replayNbr.getNbr().getAddressFamilies();
2035                 if (afs != null) {
2036                     for (AddressFamilies af : afs) {
2037                         af_afi afi = af_afi.findByValue(af.getAfi().intValue());
2038                         af_safi safi = af_safi.findByValue(af.getSafi().intValue());
2039                         replayDone = false;
2040                         try {
2041                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
2042                             replayDone = true;
2043                         } catch (TException | BgpRouterException eAFs) {
2044                             LOG.debug("Replaying AddressFamily for Nbr {}, exception:",
2045                                     replayNbr.getNbr().getAddress().getValue(), eAFs);
2046                         }
2047                         replaySuccess = replaySuccess && replayDone;
2048                     }
2049                 }
2050                 //replay is success --> no need to replay this nbr in next iteration.
2051                 replayNbr.setShouldRetry(replaySuccess ? false : true);
2052             }
2053         } while (nbrRetry.decrementAndRetry());
2054         boolean replaySuccess = true;
2055         for (ReplayNbr replayNbr : replayNbrList) {
2056             replaySuccess = replaySuccess && !replayNbr.isShouldRetry();
2057         }
2058         return replaySuccess;
2059     }
2060
2061     public String getConfigHost() {
2062         if (config == null) {
2063             return hostStartup;
2064         }
2065         ConfigServer ts = config.getConfigServer();
2066         return ts == null ? hostStartup : ts.getHost().getValue();
2067     }
2068
2069     public int getConfigPort() {
2070         if (config == null) {
2071             return Integer.parseInt(portStartup);
2072         }
2073         ConfigServer ts = config.getConfigServer();
2074         return ts == null ? Integer.parseInt(portStartup) :
2075                 ts.getPort().intValue();
2076     }
2077
2078     public Bgp getConfig() {
2079         AtomicInteger bgpDSretryCount = new AtomicInteger(DS_RETRY_COUNT);
2080         while (0 != bgpDSretryCount.decrementAndGet()) {
2081             try {
2082                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
2083                         InstanceIdentifier.create(Bgp.class)).orNull();
2084             } catch (ReadFailedException e) {
2085                 //Config DS may not be up, so sleep for 1 second and retry
2086                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
2087                 try {
2088                     Thread.sleep(WAIT_TIME_BETWEEN_EACH_TRY_MILLIS);
2089                 } catch (InterruptedException timerEx) {
2090                     LOG.debug("WAIT_TIME_BETWEEN_EACH_TRY_MILLIS, Timer got interrupted while waiting for"
2091                             + "config DS availability", timerEx);
2092                 }
2093             }
2094         }
2095         LOG.error("failed to get bgp config");
2096         return null;
2097     }
2098
2099     @SuppressWarnings("checkstyle:IllegalCatch")
2100     public synchronized boolean replay() {
2101         boolean replaySucceded = true;
2102         String host = getConfigHost();
2103         int port = getConfigPort();
2104         LOG.error("connecting  to bgp host {} ", host);
2105         boolean res = bgpRouter.connect(host, port);
2106         if (!res) {
2107             LOG.error("Cannot connect to BGP config server at {}:{}{}", host, port,
2108                     config != null ? "; Configuration Replay aborted" : "");
2109             return replaySucceded;
2110         }
2111         config = getConfig();
2112         if (config == null) {
2113             LOG.error("bgp config is empty nothing to push to bgp");
2114             return replaySucceded;
2115         }
2116         BgpRouter br = bgpRouter;
2117         AsId asId = config.getAsId();
2118         if (asId == null) {
2119             LOG.error("bgp as-id is null");
2120             return replaySucceded;
2121         }
2122         long asNum = asId.getLocalAs();
2123         IpAddress routerId = asId.getRouterId();
2124         String rid = routerId == null ? "" : routerId.stringValue();
2125         int stalepathTime = (int) getStalePathtime(RESTART_DEFAULT_GR, config.getAsId());
2126         boolean announceFbit = true;
2127         boolean replayDone = false;
2128         final int numberOfStartBgpRetries = 3;
2129         RetryOnException startBgpRetry = new RetryOnException(numberOfStartBgpRetries);
2130         do {
2131             try {
2132                 LOG.debug("Replaying BGPConfig ");
2133                 br.startBgp(asNum, rid, stalepathTime, announceFbit);
2134                 LOG.debug("Replay BGPConfig successful");
2135                 replayDone = true;
2136                 break;
2137             } catch (BgpRouterException bre) {
2138                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
2139                     LOG.debug("Starting the routesync for exception", bre);
2140                     startBgpRetry.errorOccured();
2141                     if (!startBgpRetry.shouldRetry()) {
2142                         LOG.debug("starting route sync for BgpRouter exception");
2143                         doRouteSync();
2144                     }
2145                 } else {
2146                     LOG.error("Replay: startBgp() received exception error {} : ",
2147                             bre.getErrorCode(), bre);
2148                     startBgpRetry.errorOccured();
2149                 }
2150             } catch (TApplicationException tae) {
2151                 if (tae.getType() == BgpRouterException.BGP_ERR_ACTIVE) {
2152                     LOG.debug("Starting the routesync for exception", tae);
2153                     startBgpRetry.errorOccured();
2154                     if (!startBgpRetry.shouldRetry()) {
2155                         LOG.debug("starting route sync for Thrift BGP_ERR_ACTIVE exception");
2156                         doRouteSync();
2157                     }
2158                 } else if (tae.getType() == BgpRouterException.BGP_ERR_COMMON_FAILURE) {
2159                     LOG.debug("Starting the routesync for AS-ID started exception", tae);
2160                     startBgpRetry.errorOccured();
2161                     if (!startBgpRetry.shouldRetry()) {
2162                         LOG.debug("starting route sync for Thrift BGP_ERR_COMMON_FAILURE exception");
2163                         doRouteSync();
2164                     }
2165                 } else {
2166                     LOG.error("Replay: startBgp() received exception type {}: ",
2167                             tae.getType(), tae);
2168                     startBgpRetry.errorOccured();
2169                 }
2170             } catch (Exception e) {
2171                 //not unusual. We may have restarted & BGP is already on
2172                 LOG.error("Replay:startBgp() received exception: ", e);
2173                 startBgpRetry.errorOccured();
2174             }
2175         } while (startBgpRetry.shouldRetry());
2176
2177         replaySucceded = replayDone;
2178
2179         startBgpCountersTask();
2180         startBgpAlarmsTask();
2181
2182         /*
2183          * commenting this due to a bug with QBGP. Will uncomment once QBGP fix is done.
2184          * This wont have any functional impacts
2185          */
2186         try {
2187             br.delayEOR(delayEorSeconds);
2188         } catch (TException | BgpRouterException e) {
2189             LOG.error("Replay: delayEOR() number of seconds to wait for EOR from ODL:", e);
2190         }
2191
2192         List<Neighbors> neighbors = config.getNeighbors();
2193         if (neighbors != null) {
2194             LOG.error("configuring existing Neighbors present for replay total neighbors {}", neighbors.size());
2195             boolean neighborConfigReplayResult = replayNbrConfig(neighbors, br);
2196             if (neighborConfigReplayResult == false) {
2197                 replaySucceded = false;
2198             }
2199         } else {
2200             LOG.error("no Neighbors present for replay config ");
2201         }
2202
2203         Logging logging = config.getLogging();
2204         if (logging != null) {
2205             try {
2206                 br.setLogging(logging.getFile(), logging.getLevel());
2207             } catch (TException | BgpRouterException e) {
2208                 LOG.error("Replay:setLogging() received exception", e);
2209             }
2210         }
2211
2212         GracefulRestart gracefulRestart = config.getGracefulRestart();
2213         if (gracefulRestart != null) {
2214             try {
2215                 br.addGracefulRestart(gracefulRestart.getStalepathTime().intValue());
2216             } catch (TException | BgpRouterException e) {
2217                 LOG.error("Replay:addGr() received exception", e);
2218             }
2219         }
2220
2221         List<Vrfs> vrfs = config.getVrfs();
2222         if (vrfs == null) {
2223             vrfs = new ArrayList<>();
2224         }
2225         for (Vrfs vrf : vrfs) {
2226             for (AddressFamiliesVrf adf : vrf.getAddressFamiliesVrf()) {
2227                 try {
2228                     br.addVrf(BgpUtil.getLayerType(adf), vrf.getRd(), vrf.getImportRts(),
2229                             vrf.getExportRts());
2230                 } catch (TException | BgpRouterException e) {
2231                     LOG.error("Replay:addVrf() received exception", e);
2232                 }
2233             }
2234         }
2235
2236         List<Networks> ln = config.getNetworks();
2237         if (ln != null) {
2238             for (Networks net : ln) {
2239                 String rd = net.getRd();
2240                 String pfxlen = net.getPrefixLen();
2241                 String nh = net.getNexthop().getValue();
2242                 Long label = net.getLabel();
2243                 int lbl = label == null ? 0 : label.intValue();
2244                 int l3vni = net.getL3vni() == null ? 0 : net.getL3vni().intValue();
2245                 int l2vni = net.getL2vni() == null ? 0 : net.getL2vni().intValue();
2246                 if (rd == null && lbl > 0) {
2247                     //LU prefix is being deleted.
2248                     rd = Integer.toString(lbl);
2249                 }
2250
2251                 BgpControlPlaneType protocolType = net.getBgpControlPlaneType();
2252                 int ethernetTag = net.getEthtag().intValue();
2253                 String esi = net.getEsi();
2254                 String macaddress = net.getMacaddress();
2255                 EncapType encapType = net.getEncapType();
2256                 String routerMac = net.getRoutermac();
2257
2258                 try {
2259                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni,
2260                             BgpUtil.convertToThriftProtocolType(protocolType),
2261                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
2262                 } catch (TException | BgpRouterException e) {
2263                     LOG.error("Replay:addPfx() received exception", e);
2264                 }
2265             }
2266         }
2267
2268
2269         List<Multipath> multipaths = config.getMultipath();
2270
2271         if (multipaths != null) {
2272             for (Multipath multipath : multipaths) {
2273                 if (multipath != null) {
2274                     af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
2275                     af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
2276
2277                     try {
2278                         if (multipath.isMultipathEnabled()) {
2279                             br.enableMultipath(afi, safi);
2280                         } else {
2281                             br.disableMultipath(afi, safi);
2282                         }
2283                     } catch (TException | BgpRouterException e) {
2284                         LOG.info("Replay:multipaths() received exception", e);
2285                     }
2286                 }
2287             }
2288         }
2289         List<VrfMaxpath> vrfMaxpaths = config.getVrfMaxpath();
2290         if (vrfMaxpaths != null) {
2291             for (VrfMaxpath vrfMaxpath : vrfMaxpaths) {
2292                 try {
2293                     br.multipaths(vrfMaxpath.getRd(), vrfMaxpath.getMaxpaths());
2294                 } catch (TException | BgpRouterException e) {
2295                     LOG.info("Replay:vrfMaxPath() received exception", e);
2296                 }
2297             }
2298         }
2299
2300         //send End of Rib Marker to Qthriftd.
2301         final int numberOfEORRetries = 3;
2302         replayDone = false;
2303         RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
2304         do {
2305             try {
2306                 br.sendEOR();
2307                 LOG.debug("Replay sendEOR {} successful");
2308                 replayDone = true;
2309                 break;
2310             } catch (Exception e) {
2311                 eorRetry.errorOccured();
2312                 LOG.error("Replay:sedEOR() received exception:", e);
2313             }
2314         } while (eorRetry.shouldRetry());
2315
2316         return replaySucceded && replayDone;
2317     }
2318
2319     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
2320         bgpUtil.update(iid, dto);
2321     }
2322
2323     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
2324         bgpUtil.delete(iid);
2325     }
2326
2327     public void startConfig(String bgpHost, int thriftPort) {
2328         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2329                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2330         InstanceIdentifier<ConfigServer> iid = iib.build();
2331         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
2332         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
2333                 .setPort((long) thriftPort).build();
2334         update(iid, dto);
2335     }
2336
2337     public void startBgp(long as, String routerId, int spt, boolean fbit) {
2338         IpAddress rid = routerId == null ? null : IpAddressBuilder.getDefaultInstance(routerId);
2339         Long staleTime = (long) spt;
2340         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2341                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2342         InstanceIdentifier<AsId> iid = iib.build();
2343         AsId dto = new AsIdBuilder().setLocalAs(as)
2344                 .setRouterId(rid)
2345                 .setStalepathTime(staleTime)
2346                 .setAnnounceFbit(fbit).build();
2347         update(iid, dto);
2348     }
2349
2350     public void addLogging(String fileName, String logLevel) {
2351         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2352                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2353         InstanceIdentifier<Logging> iid = iib.build();
2354         Logging dto = new LoggingBuilder().setFile(fileName)
2355                 .setLevel(logLevel).build();
2356         update(iid, dto);
2357     }
2358
2359     public void addGracefulRestart(int staleTime) {
2360         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2361                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
2362         InstanceIdentifier<GracefulRestart> iid = iib.build();
2363         GracefulRestart dto = new GracefulRestartBuilder()
2364                 .setStalepathTime((long) staleTime).build();
2365         update(iid, dto);
2366     }
2367
2368     public void addNeighbor(
2369             String nbrIp, long remoteAs, @Nullable final TcpMd5SignaturePasswordType md5Secret) {
2370         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2371         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2372                 InstanceIdentifier.builder(Bgp.class)
2373                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2374         InstanceIdentifier<Neighbors> iid = iib.build();
2375         TcpSecurityOption tcpSecOption = null;
2376         if (md5Secret != null) {
2377             tcpSecOption = new TcpMd5SignatureOptionBuilder().setTcpMd5SignaturePassword(md5Secret).build();
2378         } // else let tcpSecOption be null
2379         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
2380                 .setRemoteAs(remoteAs).setTcpSecurityOption(tcpSecOption).build();
2381         update(iid, dto);
2382     } // public addNeighbor(nbrIp, remoteAs, md5Secret)
2383
2384     public void addUpdateSource(String nbrIp, String srcIp) {
2385         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2386         Ipv4Address srcAddr = new Ipv4Address(srcIp);
2387         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2388                 InstanceIdentifier.builder(Bgp.class)
2389                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2390                         .child(UpdateSource.class);
2391         InstanceIdentifier<UpdateSource> iid = iib.build();
2392         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
2393                 .setSourceIp(srcAddr).build();
2394         update(iid, dto);
2395     }
2396
2397     public void addEbgpMultihop(String nbrIp, int hops) {
2398         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2399         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2400                 InstanceIdentifier.builder(Bgp.class)
2401                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2402                         .child(EbgpMultihop.class);
2403         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2404         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
2405                 .setNhops((long) hops).build();
2406         update(iid, dto);
2407     }
2408
2409     public void addAddressFamily(String nbrIp, int afi, int safi) {
2410         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2411         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2412                 InstanceIdentifier.builder(Bgp.class)
2413                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2414                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2415         InstanceIdentifier<AddressFamilies> iid = iib.build();
2416         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
2417                 .setAfi((long) afi).setSafi((long) safi).build();
2418         update(iid, dto);
2419     }
2420
2421     public void addPrefix(String rd, String macAddress, String pfx, List<String> nhList,
2422               VrfEntry.EncapType encapType, long lbl, long l3vni, long l2vni, String gatewayMac) {
2423         for (String nh : nhList) {
2424             Ipv4Address nexthop = nh != null ? new Ipv4Address(nh) : null;
2425             Long label = lbl;
2426             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
2427                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
2428             NetworksBuilder networksBuilder = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
2429                                                 .setLabel(label).setEthtag(BgpConstants.DEFAULT_ETH_TAG);
2430             buildVpnEncapSpecificInfo(networksBuilder, encapType, label, l3vni, l2vni, macAddress, gatewayMac);
2431             update(iid, networksBuilder.build());
2432         }
2433     }
2434
2435     private static void buildVpnEncapSpecificInfo(NetworksBuilder builder, VrfEntry.EncapType encapType, long label,
2436                                                   long l3vni, long l2vni, String macAddress, String gatewayMac) {
2437         if (encapType.equals(VrfEntry.EncapType.Mplsgre)) {
2438             builder.setLabel(label).setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLL3VPN)
2439                     .setEncapType(EncapType.GRE);
2440         } else {
2441             builder.setL3vni(l3vni).setL2vni(l2vni).setMacaddress(macAddress).setRoutermac(gatewayMac)
2442                     .setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLEVPN).setEncapType(EncapType.VXLAN);
2443         }
2444     }
2445
2446     // TODO: add LayerType as arg - supports command
2447     public void addVrf(String rd, List<String> irts, List<String> erts, AddressFamily addressFamily) {
2448         Vrfs vrf = bgpUtil.getVrfFromRd(rd);
2449         List<AddressFamiliesVrf> adfList = new ArrayList<>(1);
2450         if (vrf != null) {
2451             adfList = vrf.getAddressFamiliesVrf();
2452         }
2453         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2454         if (addressFamily.equals(AddressFamily.IPV4)) {
2455             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2456             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2457         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2458             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2459             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2460         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2461             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2462             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2463         }
2464         AddressFamiliesVrf adf = adfBuilder.build();
2465         adfList.add(adf);
2466         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib = InstanceIdentifier.builder(Bgp.class)
2467                 .child(Vrfs.class, new VrfsKey(rd));
2468         InstanceIdentifier<Vrfs> iid = iib.build();
2469         Vrfs dto = new VrfsBuilder().setRd(rd).setImportRts(irts)
2470             .setExportRts(erts).setAddressFamiliesVrf(adfList).build();
2471
2472         List<AddressFamiliesVrf> listAdFamilies = mapNewAdFamily.get(rd);
2473         if (listAdFamilies != null) {
2474             listAdFamilies.add(adf);
2475         } else {
2476             mapNewAdFamily.put(rd, adfList);
2477         }
2478
2479         try {
2480             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
2481         } catch (TransactionCommitFailedException e) {
2482             LOG.error("Error adding VRF to datastore", e);
2483             throw new RuntimeException(e);
2484         }
2485     }
2486
2487     public void stopConfig() {
2488         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2489                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2490         InstanceIdentifier<ConfigServer> iid = iib.build();
2491         delete(iid);
2492     }
2493
2494     public void stopBgp() {
2495         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2496                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2497         InstanceIdentifier<AsId> iid = iib.build();
2498         delete(iid);
2499     }
2500
2501     public void delLogging() {
2502         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2503                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2504         InstanceIdentifier<Logging> iid = iib.build();
2505         delete(iid);
2506     }
2507
2508     public void delGracefulRestart() {
2509         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2510                 InstanceIdentifier.builder(Bgp.class)
2511                         .child(GracefulRestart.class);
2512         InstanceIdentifier<GracefulRestart> iid = iib.build();
2513         delete(iid);
2514     }
2515
2516     public void delNeighbor(String nbrIp) {
2517         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2518         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2519                 InstanceIdentifier.builder(Bgp.class)
2520                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2521         InstanceIdentifier<Neighbors> iid = iib.build();
2522         delete(iid);
2523     }
2524
2525     public void delUpdateSource(String nbrIp) {
2526         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2527         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2528                 InstanceIdentifier.builder(Bgp.class)
2529                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2530                         .child(UpdateSource.class);
2531         InstanceIdentifier<UpdateSource> iid = iib.build();
2532         delete(iid);
2533     }
2534
2535     public void delEbgpMultihop(String nbrIp) {
2536         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2537         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2538                 InstanceIdentifier.builder(Bgp.class)
2539                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2540                         .child(EbgpMultihop.class);
2541         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2542         delete(iid);
2543     }
2544
2545     public void delAddressFamily(String nbrIp, int afi, int safi) {
2546         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2547         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2548                 InstanceIdentifier.builder(Bgp.class)
2549                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2550                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2551         InstanceIdentifier<AddressFamilies> iid = iib.build();
2552         delete(iid);
2553     }
2554
2555     public void delPrefix(String rd, String pfx) {
2556         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
2557                 InstanceIdentifier.builder(Bgp.class)
2558                         .child(Networks.class, new NetworksKey(pfx, rd));
2559         InstanceIdentifier<Networks> iid = iib.build();
2560         delete(iid);
2561     }
2562
2563     public boolean delVrf(String rd, AddressFamily addressFamily) {
2564         if (addressFamily == null) {
2565             LOG.error("delVrf: vrf {}, addressFamily invalid", rd);
2566             return false;
2567         }
2568         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2569         if (addressFamily.equals(AddressFamily.IPV4)) {
2570             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2571             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2572         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2573             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2574             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2575         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2576             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2577             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2578         }
2579         Vrfs vrfOriginal = bgpUtil.getVrfFromRd(rd);
2580         if (vrfOriginal == null) {
2581             LOG.error("delVrf: no vrf with existing rd {}. step aborted", rd);
2582             return false;
2583         }
2584
2585         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2586                 InstanceIdentifier.builder(Bgp.class)
2587                         .child(Vrfs.class, new VrfsKey(rd));
2588
2589         InstanceIdentifier<Vrfs> iid = iib.build();
2590
2591         InstanceIdentifier.builder(Bgp.class).build()
2592                 .child(Multipath.class, new MultipathKey(adfBuilder.getAfi(), adfBuilder.getSafi()));
2593         @SuppressWarnings("static-access")
2594         InstanceIdentifier<Bgp> iid6 =  InstanceIdentifier.create(Bgp.class);
2595         InstanceIdentifierBuilder<Vrfs> iib3 = iid6.child(Vrfs.class, new VrfsKey(rd)).builder();
2596         InstanceIdentifier<Vrfs> iidFinal = iib3.build();
2597
2598         //** update or delete the vrfs with the rest of AddressFamilies already present in the last list
2599         AddressFamiliesVrf adfToDel = adfBuilder.build();
2600         List<AddressFamiliesVrf> adfListOriginal = vrfOriginal.getAddressFamiliesVrf() == null
2601                 ? new ArrayList<>() : vrfOriginal.getAddressFamiliesVrf();
2602         List<AddressFamiliesVrf> adfListToRemoveFromOriginal = new ArrayList<>();
2603         adfListOriginal.forEach(adf -> {
2604             if (adf.equals(adfToDel)) {
2605                 adfListToRemoveFromOriginal.add(adfToDel);
2606                 return;
2607             }
2608         });
2609         for (AddressFamiliesVrf adfToRemove : adfListToRemoveFromOriginal) {
2610             adfListOriginal.remove(adfToRemove);
2611             try {
2612                 SingleTransactionDataBroker.syncWrite(dataBroker,
2613                         LogicalDatastoreType.CONFIGURATION, iid, vrfOriginal);
2614             } catch (TransactionCommitFailedException e) {
2615                 LOG.error("delVrf: Error updating VRF to datastore", e);
2616                 throw new RuntimeException(e);
2617             }
2618         }
2619         if (adfListOriginal.isEmpty()) {
2620             delete(iidFinal);
2621             return true;
2622         }
2623         // not all is removed
2624         return false;
2625     }
2626
2627     public void setMultipathStatus(af_afi afi, af_safi safi, boolean enable) {
2628         long lafi = afi.getValue();
2629         long lsafi = safi.getValue();
2630
2631         InstanceIdentifier.InstanceIdentifierBuilder<Multipath> iib =
2632                 InstanceIdentifier
2633                         .builder(Bgp.class)
2634                         .child(Multipath.class,
2635                                 new MultipathKey(Long.valueOf(afi.getValue()), Long.valueOf(safi.getValue())));
2636
2637         Multipath dto = new MultipathBuilder().setAfi(lafi).setSafi(lsafi).setMultipathEnabled(enable).build();
2638         update(iib.build(), dto);
2639     }
2640
2641     public void multipaths(String rd, int maxpath) {
2642         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
2643                 InstanceIdentifier
2644                         .builder(Bgp.class)
2645                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
2646
2647         VrfMaxpath dto = new VrfMaxpathBuilder().setRd(rd).setMaxpaths(maxpath).build();
2648         update(iib.build(), dto);
2649     }
2650
2651     /*
2652     * Remove Stale Marked Routes after timer expiry.
2653     */
2654     private class RouteCleanup implements Callable<Integer> {
2655
2656         @Override
2657         public Integer call() {
2658             totalCleared = 0;
2659             try {
2660                 if (staledFibEntriesMap.isEmpty()) {
2661                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
2662                 } else {
2663                     for (String rd : staledFibEntriesMap.keySet()) {
2664                         if (Thread.interrupted()) {
2665                             return 0;
2666                         }
2667                         Map<String, Long> map = staledFibEntriesMap.get(rd);
2668                         if (map != null) {
2669                             for (String key : map.keySet()) {
2670                                 if (Thread.interrupted()) {
2671                                     return 0;
2672                                 }
2673                                 String prefix = extractPrefix(key);
2674                                 String nextHop = extractNextHop(key);
2675                                 totalCleared++;
2676                                 LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}, nextHop:{}",
2677                                         rd, prefix, nextHop);
2678                                 fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix, nextHop);
2679                             }
2680                         }
2681                     }
2682                 }
2683             } finally {
2684                 staledFibEntriesMap.clear();
2685             }
2686             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
2687             return 0;
2688         }
2689     }
2690
2691     /*
2692      * BGP restart scenario, ODL-BGP manager was/is running.
2693      * On re-sync notification, Get a copy of FIB database.
2694      */
2695     public void createStaleFibMap() {
2696         totalStaledCount = 0;
2697         try {
2698             staledFibEntriesMap.clear();
2699             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2700
2701             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2702                     LogicalDatastoreType.CONFIGURATION, id);
2703             if (fibEntries.isPresent()) {
2704                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2705                 for (VrfTables vrfTable : staleVrfTables) {
2706                     Map<String, Long> staleFibEntMap = new HashMap<>();
2707                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2708                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2709                             //Stale marking and cleanup is only meant for the routes learned through BGP.
2710                             continue;
2711                         }
2712                         if (Thread.interrupted()) {
2713                             break;
2714                         }
2715                         totalStaledCount++;
2716                         //Create MAP from staleVrfTables.
2717                         vrfEntry.getRoutePaths()
2718                                 .forEach(
2719                                     routePath -> staleFibEntMap.put(
2720                                             appendNextHopToPrefix(vrfEntry.getDestPrefix(),
2721                                                     routePath.getNexthopAddress()), routePath.getLabel()));
2722                     }
2723                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), staleFibEntMap);
2724                 }
2725             } else {
2726                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
2727             }
2728         } catch (ReadFailedException e) {
2729             LOG.error("createStaleFibMap:: error ", e);
2730         }
2731         LOG.error("created {} staled entries ", totalStaledCount);
2732     }
2733
2734     /*
2735      * BGP config remove scenario, Need to remove all the
2736      * external routes from FIB.
2737      */
2738     public void deleteExternalFibRoutes() {
2739         totalExternalRoutes = 0;
2740         totalExternalMacRoutes = 0;
2741         try {
2742             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2743
2744             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2745                     LogicalDatastoreType.CONFIGURATION, id);
2746             if (fibEntries.isPresent()) {
2747                 if (fibEntries.get().getVrfTables() == null) {
2748                     LOG.error("deleteExternalFibRoutes::getVrfTables is null");
2749                     return;
2750                 }
2751                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2752                 for (VrfTables vrfTable : staleVrfTables) {
2753                     String rd = vrfTable.getRouteDistinguisher();
2754                     if (vrfTable.getVrfEntry() != null) {
2755                         for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2756                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2757                                 //route cleanup is only meant for the routes learned through BGP.
2758                                 continue;
2759                             }
2760                             totalExternalRoutes++;
2761                             fibDSWriter.removeFibEntryFromDS(rd, vrfEntry.getDestPrefix());
2762                         }
2763                     } else if (vrfTable.getMacVrfEntry() != null) {
2764                         for (MacVrfEntry macEntry : vrfTable.getMacVrfEntry()) {
2765                             if (RouteOrigin.value(macEntry.getOrigin()) != RouteOrigin.BGP) {
2766                                 //route cleanup is only meant for the routes learned through BGP.
2767                                 continue;
2768                             }
2769                             totalExternalMacRoutes++;
2770                             fibDSWriter.removeMacEntryFromDS(rd, macEntry.getMac());
2771                         }
2772                     }
2773                 }
2774             } else {
2775                 LOG.error("deleteExternalFibRoutes:: FIBentries.class is not present");
2776             }
2777         } catch (ReadFailedException e) {
2778             LOG.error("deleteExternalFibRoutes:: error ", e);
2779         }
2780         LOG.debug("deleted {} fib entries {} mac entries", totalExternalRoutes, totalExternalMacRoutes);
2781     }
2782
2783     public boolean addToRt2TepMap(String rd, String tepIp, String mac, Long l2vni) {
2784         boolean isFirstMacUpdateFromTep = false;
2785         if (rt2TepMap.containsKey(rd)) {
2786             if (rt2TepMap.get(rd).containsKey(tepIp)) {
2787                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and tep-ip {}. No Elan DS write required",
2788                         mac, l2vni, rd, tepIp);
2789                 rt2TepMap.get(rd).get(tepIp).put(mac, l2vni);
2790             } else {
2791                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and new tep-ip {}",
2792                         mac, l2vni, rd, tepIp);
2793                 isFirstMacUpdateFromTep = true;
2794                 Map<String, Long> macList = new HashMap<>();
2795                 macList.put(mac, l2vni);
2796                 rt2TepMap.get(rd).put(tepIp, macList);
2797             }
2798         } else {
2799             LOG.debug("RT2 with mac {} l2vni {} from new rd {} and tep ip {}",
2800                     mac, l2vni, rd, tepIp);
2801             isFirstMacUpdateFromTep = true;
2802             Map<String, Long> macList = new HashMap<>();
2803             macList.put(mac, l2vni);
2804             Map<String, Map<String, Long>> tepIpMacMap = new HashMap<>();
2805             tepIpMacMap.put(tepIp, macList);
2806             rt2TepMap.put(rd, tepIpMacMap);
2807         }
2808         return isFirstMacUpdateFromTep;
2809     }
2810
2811     public boolean deleteFromRt2TepMap(String rd, String tepIp, String mac) {
2812         boolean isLastMacUpdateFromTep = false;
2813         LOG.debug("RT2 withdraw with rd {} mac {} tep-ip {} ", rd, mac, tepIp);
2814         if (rt2TepMap.containsKey(rd)) {
2815             if (rt2TepMap.get(rd).containsKey(tepIp)) {
2816                 if (rt2TepMap.get(rd).get(tepIp).containsKey(mac)) {
2817                     LOG.debug("RT2 Withdraw : Removing the mac {} from Map", mac);
2818                     rt2TepMap.get(rd).get(tepIp).remove(mac);
2819                     if (rt2TepMap.get(rd).get(tepIp).isEmpty()) {
2820                         isLastMacUpdateFromTep = true;
2821                         LOG.debug("RT2 Withdraw : Removing the tep-ip {} from Map", tepIp);
2822                         rt2TepMap.get(rd).remove(tepIp);
2823                         if (rt2TepMap.get(rd).isEmpty()) {
2824                             LOG.debug("RT2 Withdraw : Removing the rd {} from Map", rd);
2825                             rt2TepMap.remove(rd);
2826                         }
2827                     }
2828                 }
2829             }
2830         }
2831         return isLastMacUpdateFromTep;
2832     }
2833
2834     public Collection<String> getTepIPs(String rd) {
2835         final Map<String, Map<String, Long>> tepIpMap = rt2TepMap.get(rd);
2836         return tepIpMap != null ? tepIpMap.keySet() : Collections.emptyList();
2837     }
2838
2839     public boolean isBgpConnected() {
2840         return bgpRouter.isBgpConnected();
2841     }
2842
2843     public long getLastConnectedTS() {
2844         return bgpRouter.getLastConnectedTS();
2845     }
2846
2847     public long getConnectTS() {
2848         return bgpRouter.getConnectTS();
2849     }
2850
2851     public long getStartTS() {
2852         return bgpRouter.getStartTS();
2853     }
2854
2855     public TTransport getTransport() {
2856         return bgpRouter.getTransport();
2857     }
2858
2859     public int getTotalStaledCount() {
2860         return totalStaledCount;
2861     }
2862
2863     public int getTotalCleared() {
2864         return totalCleared;
2865     }
2866
2867     public BgpCounters getBgpCounters() {
2868         return bgpCountersReference.get();
2869     }
2870
2871     private void startBgpCountersTask() {
2872         if (getBgpCounters() == null && bgpCountersReference.compareAndSet(null,
2873                 new BgpCounters(getBgpSdncMipIp(), metricProvider))) {
2874             bgpCountersTask = executor.scheduleAtFixedRate(bgpCountersReference.get(), 0, 120 * 1000,
2875                     TimeUnit.MILLISECONDS);
2876             LOG.info("Bgp Counters task scheduled for every two minutes.");
2877
2878             bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
2879         }
2880     }
2881
2882     private void stopBgpCountersTask() {
2883         final BgpCounters bgpCounters = bgpCountersReference.getAndSet(null);
2884         if (bgpCounters != null) {
2885             bgpCountersTask.cancel(true);
2886             bgpCounters.close();
2887         }
2888     }
2889
2890     private void startBgpAlarmsTask() {
2891         if (getBgpAlarms() == null && bgpAlarmsReference.compareAndSet(null, new BgpAlarms(this))) {
2892             bgpAlarmsReference.get().init();
2893             bgpAlarmsTask = executor.scheduleAtFixedRate(bgpAlarmsReference.get(), 0, 60 * 1000, TimeUnit.MILLISECONDS);
2894             LOG.info("Bgp Alarms task scheduled for every minute.");
2895         }
2896     }
2897
2898     private void stopBgpAlarmsTask() {
2899         final BgpAlarms bgpAlarms = bgpAlarmsReference.getAndSet(null);
2900         if (bgpAlarms != null) {
2901             bgpAlarmsTask.cancel(true);
2902             bgpAlarms.close();
2903         }
2904     }
2905
2906     public BgpAlarms getBgpAlarms() {
2907         return bgpAlarmsReference.get();
2908     }
2909
2910     private static String appendNextHopToPrefix(String prefix, String nextHop) {
2911         return prefix + ":" + nextHop;
2912     }
2913
2914     private static String extractPrefix(String prefixNextHop) {
2915         return prefixNextHop.split(":")[0];
2916     }
2917
2918     private static String extractNextHop(String prefixNextHop) {
2919         return prefixNextHop.split(":")[1];
2920     }
2921
2922     private static String extractMd5Secret(final Neighbors val) {
2923         String md5Secret = null;
2924         TcpSecurityOption tcpSecOpt = val.getTcpSecurityOption();
2925         if (tcpSecOpt != null) {
2926             if (tcpSecOpt instanceof TcpMd5SignatureOption) {
2927                 md5Secret = ((TcpMd5SignatureOption) tcpSecOpt).getTcpMd5SignaturePassword().getValue();
2928             } else { // unknown TcpSecurityOption
2929                 LOG.debug("neighbors  Ignored unknown tcp-security-option of peer {}", val.getAddress().getValue());
2930             }
2931         }
2932         return md5Secret;
2933     } // private method extractMd5Secret
2934 }