bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.net.InetAddresses;
  13 import com.google.common.util.concurrent.ThreadFactoryBuilder;
  14 import io.netty.util.concurrent.GlobalEventExecutor;
  15 import java.io.IOException;
  16 import java.lang.reflect.Constructor;
  17 import java.lang.reflect.InvocationTargetException;
  18 import java.net.InetAddress;
  19 import java.net.InetSocketAddress;
  20 import java.net.NetworkInterface;
  21 import java.net.SocketException;
  22 import java.util.ArrayList;
  23 import java.util.Collection;
  24 import java.util.Collections;
  25 import java.util.Enumeration;
  26 import java.util.HashMap;
  27 import java.util.Iterator;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Timer;
  32 import java.util.concurrent.Callable;
  33 import java.util.concurrent.ConcurrentHashMap;
  34 import java.util.concurrent.CountDownLatch;
  35 import java.util.concurrent.ExecutionException;
  36 import java.util.concurrent.Executors;
  37 import java.util.concurrent.Future;
  38 import java.util.concurrent.ScheduledExecutorService;
  39 import java.util.concurrent.ScheduledFuture;
  40 import java.util.concurrent.TimeUnit;
  41 import java.util.concurrent.TimeoutException;
  42 import java.util.concurrent.atomic.AtomicInteger;
  43 import java.util.concurrent.atomic.AtomicReference;
  44 import javax.annotation.Nullable;
  45 import javax.annotation.PreDestroy;
  46 import javax.inject.Inject;
  47 import javax.inject.Singleton;
  48 import org.apache.thrift.TApplicationException;
  49 import org.apache.thrift.TException;
  50 import org.apache.thrift.transport.TTransport;
  51 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  52 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  53 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  54 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  55 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  56 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  57 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  60 import org.opendaylight.infrautils.metrics.MetricProvider;
  61 import org.opendaylight.mdsal.eos.binding.api.Entity;
  62 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipCandidateRegistration;
  63 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipListenerRegistration;
  64 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  65 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  66 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  67 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  68 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  69 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  70 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  71 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  72 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  73 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  74 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  75 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  76 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  77 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  78 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  79 import org.opendaylight.netvirt.bgpmanager.thrift.gen.protocol_type;
  80 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  81 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  82 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  83 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  84 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.AddressFamily;
  85 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  86 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
  87 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EncapType;
  88 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.TcpMd5SignaturePasswordType;
  89 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
  90 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
  92 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
  94 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
  96 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Multipath;
  98 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathKey;
 100 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Neighbors;
 101 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsKey;
 103 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Networks;
 104 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksKey;
 106 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpath;
 107 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathKey;
 109 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Vrfs;
 110 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsKey;
 112 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamilies;
 113 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesKey;
 115 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihop;
 116 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihopBuilder;
 117 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSource;
 118 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSourceBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfs.AddressFamiliesVrf;
 120 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfs.AddressFamiliesVrfBuilder;
 121 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.TcpSecurityOption;
 122 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOption;
 123 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOptionBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
 125 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.macvrfentries.MacVrfEntry;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 132 import org.opendaylight.yangtools.yang.binding.DataObject;
 133 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 134 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 135 import org.osgi.framework.BundleContext;
 136 import org.osgi.util.tracker.ServiceTracker;
 137 import org.slf4j.Logger;
 138 import org.slf4j.LoggerFactory;
 139
 140 @Singleton
 141 public class BgpConfigurationManager {
 142     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 143
 144     // to have stale FIB map (RD, Prefix)
 145     //  number of seconds wait for route sync-up between ODL and BGP
 146     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 147     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 148     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 149     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 150     private static final String DEF_UPORT = "6644";
 151     private static final String DEF_CHOST = "255.255.255.255"; // Invalid Host IP
 152     private static final String DEF_CPORT = "7644";
 153     private static final String DEF_BGP_SDNC_MIP = "127.0.0.1";
 154     private static final String BGP_SDNC_MIP = "vpnservice.bgp.thrift.sdnc.mip";
 155     private static final int RESTART_DEFAULT_GR = 90;
 156     private static final int DS_RETRY_COUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
 157     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
 158     private static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 159     private static final String BGP_EOR_DELAY = "vpnservice.bgp.eordelay";
 160     private static final String DEF_BGP_EOR_DELAY = "1800";
 161     private static final String BGP_ENTITY_NAME = "bgp";
 162     private static final String ADD_WARN = "Config store updated; undo with Delete if needed.";
 163     private static final String DEL_WARN = "Config store updated; undo with Add if needed.";
 164     private static final String UPD_WARN = "Update operation not supported; Config store updated;"
 165             + " restore with another Update if needed.";
 166
 167     private static final Class<?>[] REACTORS = {
 168         ConfigServerReactor.class, AsIdReactor.class,
 169         GracefulRestartReactor.class, LoggingReactor.class,
 170         NeighborsReactor.class, UpdateSourceReactor.class,
 171         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 172         NetworksReactor.class, VrfsReactor.class, BgpReactor.class,
 173         MultipathReactor.class, VrfMaxpathReactor.class
 174     };
 175
 176     private IBgpManager bgpManager;
 177     private final DataBroker dataBroker;
 178     private final FibDSWriter fibDSWriter;
 179     private final IVpnLinkService vpnLinkService;
 180     private final BundleContext bundleContext;
 181     private final BgpUtil bgpUtil;
 182     private volatile Bgp config;
 183     private final BgpRouter bgpRouter;
 184     private final BgpSyncHandle bgpSyncHandle = new BgpSyncHandle();
 185     private volatile BgpThriftService bgpThriftService = null;
 186     private final int delayEorSeconds;
 187
 188     private final CountDownLatch initer = new CountDownLatch(1);
 189
 190     private final String hostStartup;
 191     private final String portStartup;
 192
 193     private final AtomicReference<BgpCounters> bgpCountersReference = new AtomicReference<>();
 194     private ScheduledFuture<?> bgpCountersTask;
 195
 196     private final AtomicReference<BgpAlarms> bgpAlarmsReference = new AtomicReference<>();
 197     private ScheduledFuture<?> bgpAlarmsTask;
 198
 199     private Future<?> lastReplayJobFt;
 200     private ScheduledFuture<?> routeCleanupFuture;
 201
 202     private long staleStartTime;
 203     private long staleEndTime;
 204     private long cfgReplayStartTime;
 205     private long cfgReplayEndTime;
 206     private long staleCleanupTime;
 207     private int totalStaledCount;
 208     private int totalCleared;
 209     private int totalExternalRoutes;
 210     private int totalExternalMacRoutes;
 211
 212     private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(
 213             new ThreadFactoryBuilder().setNameFormat("BgpConfigurationManager-%d").setDaemon(true).build());
 214
 215     /**
 216      * this map store the new address families to send to quagga. When it is sended you must clear it.
 217      * The keys String are rd (route distinguisher).
 218      * */
 219     private final ConcurrentHashMap<String, List<AddressFamiliesVrf>> mapNewAdFamily = new ConcurrentHashMap<>();
 220
 221     // map<rd, map<prefix/len:nexthop, label>>
 222     private final Map<String, Map<String, Long>> staledFibEntriesMap = new ConcurrentHashMap<>();
 223
 224     // map<rd, map<tep-ip, map<mac, l2vni>>>
 225     private final Map<String, Map<String, Map<String, Long>>> rt2TepMap = new ConcurrentHashMap<>();
 226
 227     private final List<AutoCloseable> listeners = new ArrayList<>();
 228
 229     private final EntityOwnershipUtils entityOwnershipUtils;
 230     private final EntityOwnershipCandidateRegistration candidateRegistration;
 231     private final EntityOwnershipListenerRegistration entityListenerRegistration;
 232     private final MetricProvider metricProvider;
 233
 234     @Inject
 235     public BgpConfigurationManager(final DataBroker dataBroker,
 236             final EntityOwnershipService entityOwnershipService,
 237             final FibDSWriter fibDSWriter,
 238             final IVpnLinkService vpnLinkSrvce,
 239             final BundleContext bundleContext,
 240             final BgpUtil bgpUtil,
 241             final MetricProvider metricProvider) {
 242         this.dataBroker = dataBroker;
 243         this.fibDSWriter = fibDSWriter;
 244         this.vpnLinkService = vpnLinkSrvce;
 245         this.bundleContext = bundleContext;
 246         this.bgpUtil = bgpUtil;
 247         this.metricProvider = metricProvider;
 248         String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
 249         hostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 250         portStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 251         LOG.info("ConfigServer at {}:{}", hostStartup, portStartup);
 252         VtyshCli.setHostAddr(hostStartup);
 253         ClearBgpCli.setHostAddr(hostStartup);
 254         bgpRouter = BgpRouter.newInstance(this::getConfig, this::isBGPEntityOwner);
 255         delayEorSeconds = Integer.parseInt(getProperty(BGP_EOR_DELAY, DEF_BGP_EOR_DELAY));
 256         registerCallbacks();
 257
 258         entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 259
 260         candidateRegistration = registerEntityCandidate(entityOwnershipService);
 261         entityListenerRegistration = registerEntityListener(entityOwnershipService);
 262
 263         LOG.info("BGP Configuration manager initialized");
 264         initer.countDown();
 265
 266         GlobalEventExecutor.INSTANCE.execute(() -> {
 267             ServiceTracker<IBgpManager, ?> tracker = null;
 268             try {
 269                 tracker = new ServiceTracker<>(bundleContext, IBgpManager.class, null);
 270                 tracker.open();
 271                 bgpManager = (IBgpManager) tracker.waitForService(TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES));
 272                 Preconditions.checkState(bgpManager != null, "IBgpManager service not found");
 273             } catch (InterruptedException e) {
 274                 throw new IllegalStateException("Error retrieving IBgpManager service", e);
 275             } finally {
 276                 if (tracker != null) {
 277                     tracker.close();
 278                 }
 279             }
 280
 281             if (InetAddresses.isInetAddress(getBgpSdncMipIp())) {
 282                 InetSocketAddress bgpThriftServerSocketAddr = new InetSocketAddress(getBgpSdncMipIp(),
 283                         Integer.parseInt(updatePort));
 284                 bgpThriftService = new BgpThriftService(bgpThriftServerSocketAddr, bgpManager, this);
 285                 if (isBGPEntityOwner()) {
 286                     //I am EoS owner of BGP, opening bgp thrift UPDATE-SERVER port.
 287                     LOG.info("BGP Configuration manager initialized: UPDATE-SERVER started");
 288                     bgpThriftService.start();
 289                 }
 290                 LOG.info("UPDATE server started :ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 291             } else {
 292                 LOG.error("Failed to init UPDATE server invalid ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 293             }
 294             LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 295         });
 296     }
 297
 298     public String getBgpSdncMipIp() {
 299         return getProperty(BGP_SDNC_MIP, DEF_BGP_SDNC_MIP);
 300     }
 301
 302     public long getStaleCleanupTime() {
 303         return staleCleanupTime;
 304     }
 305
 306     public void setStaleCleanupTime(long staleCleanupTime) {
 307         this.staleCleanupTime = staleCleanupTime;
 308     }
 309
 310     public long getCfgReplayEndTime() {
 311         return cfgReplayEndTime;
 312     }
 313
 314     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 315         this.cfgReplayEndTime = cfgReplayEndTime;
 316     }
 317
 318     public long getCfgReplayStartTime() {
 319         return cfgReplayStartTime;
 320     }
 321
 322     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 323         this.cfgReplayStartTime = cfgReplayStartTime;
 324     }
 325
 326     public long getStaleEndTime() {
 327         return staleEndTime;
 328     }
 329
 330     public void setStaleEndTime(long staleEndTime) {
 331         this.staleEndTime = staleEndTime;
 332     }
 333
 334     public long getStaleStartTime() {
 335         return staleStartTime;
 336     }
 337
 338     public void setStaleStartTime(long staleStartTime) {
 339         this.staleStartTime = staleStartTime;
 340     }
 341
 342     private Object createListener(Class<?> cls) {
 343         try {
 344             Constructor<?> ctor = cls.getConstructor(BgpConfigurationManager.class);
 345             return ctor.newInstance(this);
 346         } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException
 347                 e) {
 348             LOG.error("Failed to create listener object", e);
 349             return null;
 350         }
 351     }
 352
 353     private void registerCallbacks() {
 354         for (Class<?> reactor : REACTORS) {
 355             Object obj = createListener(reactor);
 356             if (obj != null) {
 357                 AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 358                 dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 359                 listeners.add(dcl);
 360             }
 361         }
 362     }
 363
 364     @SuppressWarnings("checkstyle:IllegalCatch")
 365     @PreDestroy
 366     public void close() {
 367         executor.shutdown();
 368
 369         if (bgpThriftService != null) {
 370             bgpThriftService.stop();
 371             bgpThriftService = null;
 372         }
 373
 374         if (isBgpConnected()) {
 375             //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 376             bgpRouter.disconnect();
 377         }
 378
 379         if (candidateRegistration != null) {
 380             candidateRegistration.close();
 381         }
 382
 383         entityListenerRegistration.close();
 384
 385         listeners.forEach(l -> {
 386             try {
 387                 l.close();
 388             } catch (Exception e) {
 389                 LOG.warn("Error closing {}", l ,e);
 390             }
 391         });
 392
 393         LOG.info("{} close", getClass().getSimpleName());
 394     }
 395
 396     private String getProperty(String var, String def) {
 397         String property = bundleContext.getProperty(var);
 398         return property == null ? def : property;
 399     }
 400
 401     private EntityOwnershipCandidateRegistration registerEntityCandidate(
 402             final EntityOwnershipService entityOwnershipService) {
 403         try {
 404             return entityOwnershipService.registerCandidate(
 405                     new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME));
 406         } catch (CandidateAlreadyRegisteredException e) {
 407             LOG.error("failed to register bgp entity", e);
 408             return null;
 409         }
 410     }
 411
 412     private EntityOwnershipListenerRegistration registerEntityListener(
 413             final EntityOwnershipService entityOwnershipService) {
 414         return entityOwnershipService.registerListener(BGP_ENTITY_TYPE_FOR_OWNERSHIP, ownershipChange -> {
 415             LOG.trace("entity owner change event fired: {}", ownershipChange);
 416
 417             if (ownershipChange.getState() == EntityOwnershipChangeState.LOCAL_OWNERSHIP_GRANTED) {
 418                 LOG.trace("This PL is the Owner");
 419                 if (bgpThriftService != null) {
 420                     //opening UPDATE-SERVER port.
 421                     bgpThriftService.start();
 422                 } else {
 423                     LOG.error("I am the owner of BGP entity, but bgpThriftService is not initialized yet");
 424                 }
 425                 bgpRestarted();
 426             } else {
 427                 LOG.debug("Not owner: hasOwner: {}, isOwner: {}", ownershipChange.getState().hasOwner(),
 428                         ownershipChange.getState().isOwner());
 429                 if (bgpThriftService != null && bgpThriftService.isBgpThriftServiceStarted()) {
 430                     //close the bgp Thrift Update-SERVER port opened on non-Entity Owner
 431                     bgpThriftService.stop();
 432                     bgpThriftService = null;
 433                 }
 434                 if (isBgpConnected()) {
 435                     //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 436                     bgpRouter.disconnect();
 437                 }
 438             }
 439         });
 440     }
 441
 442     public boolean isBGPEntityOwner() {
 443         return entityOwnershipUtils.isEntityOwner(new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME), 0, 1);
 444     }
 445
 446     public Bgp get() {
 447         config = getConfig();
 448         return config;
 449     }
 450
 451     public class ConfigServerReactor
 452             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 453             implements ClusteredDataTreeChangeListener<ConfigServer> {
 454         private static final String YANG_OBJ = "config-server ";
 455
 456         public ConfigServerReactor() {
 457             super(ConfigServer.class, ConfigServerReactor.class);
 458         }
 459
 460         @Override
 461         protected void add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 462             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 463             if (!isBGPEntityOwner()) {
 464                 return;
 465             }
 466
 467             try {
 468                 initer.await();
 469             } catch (InterruptedException e) {
 470                 // Ignored
 471             }
 472             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 473             bgpRouter.configServerUpdated();
 474             synchronized (BgpConfigurationManager.this) {
 475                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 476                         val.getPort().intValue());
 477                 if (!res) {
 478                     LOG.error(YANG_OBJ + "Add failed; " + ADD_WARN);
 479                 }
 480             }
 481             VtyshCli.setHostAddr(val.getHost().getValue());
 482             ClearBgpCli.setHostAddr(val.getHost().getValue());
 483         }
 484
 485         @Override
 486         protected ConfigServerReactor getDataTreeChangeListener() {
 487             return ConfigServerReactor.this;
 488         }
 489
 490         @Override
 491         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 492             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 493         }
 494
 495         @Override
 496         protected void remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 497             LOG.trace("received bgp disconnect");
 498             if (!isBGPEntityOwner()) {
 499                 return;
 500             }
 501
 502             bgpRouter.configServerUpdated();
 503
 504             synchronized (BgpConfigurationManager.this) {
 505                 bgpRouter.disconnect();
 506             }
 507         }
 508
 509         @Override
 510         protected void update(InstanceIdentifier<ConfigServer> iid,
 511                 ConfigServer oldval, ConfigServer newval) {
 512             LOG.trace("received bgp Connection update");
 513             if (!isBGPEntityOwner()) {
 514                 return;
 515             }
 516             LOG.error(YANG_OBJ + UPD_WARN);
 517         }
 518     }
 519
 520     private BgpRouter getClient(String yangObj) {
 521         if (bgpRouter == null || !bgpRouter.isBgpConnected()) {
 522             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 523             return null;
 524         }
 525         return bgpRouter;
 526     }
 527
 528     public class AsIdReactor
 529             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 530             implements ClusteredDataTreeChangeListener<AsId> {
 531
 532         private static final String YANG_OBJ = "as-id ";
 533
 534         public AsIdReactor() {
 535             super(AsId.class, AsIdReactor.class);
 536         }
 537
 538         @Override
 539         protected void add(InstanceIdentifier<AsId> iid, AsId val) {
 540             LOG.error("received bgp add asid {}", val);
 541             if (!isBGPEntityOwner()) {
 542                 return;
 543             }
 544             LOG.debug("received add router config asNum {}", val.getLocalAs());
 545             synchronized (BgpConfigurationManager.this) {
 546                 BgpRouter br = getClient(YANG_OBJ);
 547                 if (br == null) {
 548                     LOG.debug("{} Unable to process add for asNum {}; {} {}", YANG_OBJ, val.getLocalAs(),
 549                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 550                     return;
 551                 }
 552                 bgpRestarted();
 553
 554                 startBgpCountersTask();
 555                 startBgpAlarmsTask();
 556             }
 557         }
 558
 559         @Override
 560         protected AsIdReactor getDataTreeChangeListener() {
 561             return AsIdReactor.this;
 562         }
 563
 564         @Override
 565         protected InstanceIdentifier<AsId> getWildCardPath() {
 566             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 567         }
 568
 569         @Override
 570         protected void remove(InstanceIdentifier<AsId> iid, AsId val) {
 571             LOG.error("received delete router config asNum {}", val.getLocalAs());
 572             if (!isBGPEntityOwner()) {
 573                 return;
 574             }
 575             synchronized (BgpConfigurationManager.this) {
 576                 long asNum = val.getLocalAs();
 577                 BgpRouter br = getClient(YANG_OBJ);
 578                 if (br == null) {
 579                     LOG.debug("{} Unable to process remove for asNum {}; {} {}", YANG_OBJ, asNum,
 580                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 581                     return;
 582                 }
 583                 try {
 584                     br.stopBgp(asNum);
 585                 } catch (TException | BgpRouterException e) {
 586                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 587                 }
 588
 589                 stopBgpCountersTask();
 590                 stopBgpAlarmsTask();
 591
 592                 Bgp conf = getConfig();
 593                 if (conf == null) {
 594                     LOG.error("Config Null while removing the as-id");
 595                     return;
 596                 }
 597                 LOG.debug("Removing external routes from FIB");
 598                 deleteExternalFibRoutes();
 599                 List<Neighbors> nbrs = conf.getNeighbors();
 600                 if (nbrs != null && nbrs.size() > 0) {
 601                     LOG.error("Tring to remove the as-id when neighbor config is already present");
 602                     for (Neighbors nbr : nbrs) {
 603                         LOG.debug("Removing Neighbor {} from Data store", nbr.getAddress().getValue());
 604                         delNeighbor(nbr.getAddress().getValue());
 605                     }
 606                 }
 607             }
 608         }
 609
 610         @Override
 611         protected void update(InstanceIdentifier<AsId> iid,
 612                 AsId oldval, AsId newval) {
 613             if (!isBGPEntityOwner()) {
 614                 return;
 615             }
 616             LOG.error(YANG_OBJ + UPD_WARN);
 617         }
 618     }
 619
 620     public class GracefulRestartReactor
 621             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 622             implements ClusteredDataTreeChangeListener<GracefulRestart> {
 623
 624         private static final String YANG_OBJ = "graceful-restart ";
 625
 626         public GracefulRestartReactor() {
 627             super(GracefulRestart.class, GracefulRestartReactor.class);
 628         }
 629
 630         @Override
 631         protected void add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 632             if (!isBGPEntityOwner()) {
 633                 return;
 634             }
 635             synchronized (BgpConfigurationManager.this) {
 636                 int stalePathTime = val.getStalepathTime().intValue();
 637                 BgpRouter br = getClient(YANG_OBJ);
 638                 if (br == null) {
 639                     LOG.error("{} Unable to add stale-path time {}; {} {}", YANG_OBJ, stalePathTime,
 640                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 641                     return;
 642                 }
 643                 try {
 644                     br.addGracefulRestart(stalePathTime);
 645                 } catch (TException | BgpRouterException e) {
 646                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 647                 }
 648             }
 649         }
 650
 651         @Override
 652         protected GracefulRestartReactor getDataTreeChangeListener() {
 653             return GracefulRestartReactor.this;
 654         }
 655
 656         @Override
 657         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 658             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 659         }
 660
 661         @Override
 662         protected void remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 663             if (!isBGPEntityOwner()) {
 664                 return;
 665             }
 666             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 667             synchronized (BgpConfigurationManager.this) {
 668                 BgpRouter br = getClient(YANG_OBJ);
 669                 if (br == null) {
 670                     LOG.error("{} Unable to delete stale-path time; {} {}", YANG_OBJ,
 671                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 672                     return;
 673                 }
 674                 try {
 675                     br.delGracefulRestart();
 676                 } catch (TException | BgpRouterException e) {
 677                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 678                 }
 679             }
 680         }
 681
 682         @Override
 683         protected void update(InstanceIdentifier<GracefulRestart> iid,
 684                 GracefulRestart oldval, GracefulRestart newval) {
 685             if (!isBGPEntityOwner()) {
 686                 return;
 687             }
 688             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 689             synchronized (BgpConfigurationManager.this) {
 690                 int stalePathTime = newval.getStalepathTime().intValue();
 691                 BgpRouter br = getClient(YANG_OBJ);
 692                 if (br == null) {
 693                     LOG.error("{} Unable to update stale-path time to {}; {} {}", YANG_OBJ, stalePathTime,
 694                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 695                     return;
 696                 }
 697                 try {
 698                     br.addGracefulRestart(stalePathTime);
 699                 } catch (TException | BgpRouterException e) {
 700                     LOG.error("{} update received exception; {}", YANG_OBJ, ADD_WARN, e);
 701                 }
 702             }
 703         }
 704     }
 705
 706     public class LoggingReactor
 707             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 708             implements ClusteredDataTreeChangeListener<Logging> {
 709
 710         private static final String YANG_OBJ = "logging ";
 711
 712         public LoggingReactor() {
 713             super(Logging.class, LoggingReactor.class);
 714         }
 715
 716         @Override
 717         protected void add(InstanceIdentifier<Logging> iid, Logging val) {
 718             if (!isBGPEntityOwner()) {
 719                 return;
 720             }
 721             synchronized (BgpConfigurationManager.this) {
 722                 BgpRouter br = getClient(YANG_OBJ);
 723                 if (br == null) {
 724                     LOG.error("{} Unable to add logging for qbgp; {} {}", YANG_OBJ,
 725                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 726                     return;
 727                 }
 728                 try {
 729                     br.setLogging(val.getFile(), val.getLevel());
 730                 } catch (TException | BgpRouterException e) {
 731                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 732                 }
 733             }
 734         }
 735
 736         @Override
 737         protected LoggingReactor getDataTreeChangeListener() {
 738             return LoggingReactor.this;
 739         }
 740
 741         @Override
 742         protected InstanceIdentifier<Logging> getWildCardPath() {
 743             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 744         }
 745
 746         @Override
 747         protected void remove(InstanceIdentifier<Logging> iid, Logging val) {
 748             if (!isBGPEntityOwner()) {
 749                 return;
 750             }
 751             LOG.debug("received remove Logging config val {}", val.getLevel());
 752             synchronized (BgpConfigurationManager.this) {
 753                 BgpRouter br = getClient(YANG_OBJ);
 754                 if (br == null) {
 755                     LOG.error("{} Unable to remove logging for qbgp; {} {}", YANG_OBJ,
 756                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 757                     return;
 758                 }
 759                 try {
 760                     br.setLogging(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
 761                 } catch (TException | BgpRouterException e) {
 762                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 763                 }
 764             }
 765         }
 766
 767         @Override
 768         protected void update(InstanceIdentifier<Logging> iid,
 769                 Logging oldval, Logging newval) {
 770             if (!isBGPEntityOwner()) {
 771                 return;
 772             }
 773             synchronized (BgpConfigurationManager.this) {
 774                 BgpRouter br = getClient(YANG_OBJ);
 775                 if (br == null) {
 776                     LOG.error("{} Unable to update logging for qbgp; {} {}", YANG_OBJ,
 777                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 778                     return;
 779                 }
 780                 try {
 781                     br.setLogging(newval.getFile(), newval.getLevel());
 782                 } catch (TException | BgpRouterException e) {
 783                     LOG.error("{} newval received exception; {}", YANG_OBJ, ADD_WARN, e);
 784                 }
 785             }
 786         }
 787     }
 788
 789     public class NeighborsReactor
 790             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 791             implements ClusteredDataTreeChangeListener<Neighbors> {
 792
 793         private static final String YANG_OBJ = "neighbors ";
 794
 795         public NeighborsReactor() {
 796             super(Neighbors.class, NeighborsReactor.class);
 797         }
 798
 799         @Override
 800         protected void add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 801             if (!isBGPEntityOwner()) {
 802                 return;
 803             }
 804             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 805             synchronized (BgpConfigurationManager.this) {
 806                 String peerIp = val.getAddress().getValue();
 807                 long as = val.getRemoteAs();
 808                 final String md5Secret = extractMd5Secret(val);
 809                 BgpRouter br = getClient(YANG_OBJ);
 810                 if (br == null) {
 811                     LOG.debug("{} Unable to process add for peer {} as {}; {} {}", YANG_OBJ, peerIp, as,
 812                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 813                     return;
 814                 }
 815                 try {
 816                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 817                     br.addNeighbor(peerIp, as, md5Secret);
 818
 819                 } catch (TException | BgpRouterException e) {
 820                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 821                 }
 822             }
 823         }
 824
 825         @Override
 826         protected NeighborsReactor getDataTreeChangeListener() {
 827             return NeighborsReactor.this;
 828         }
 829
 830         @Override
 831         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 832             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class);
 833         }
 834
 835         @Override
 836         protected void remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 837             if (!isBGPEntityOwner()) {
 838                 return;
 839             }
 840             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 841             synchronized (BgpConfigurationManager.this) {
 842                 String peerIp = val.getAddress().getValue();
 843                 BgpRouter br = getClient(YANG_OBJ);
 844                 if (br == null) {
 845                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 846                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 847                     return;
 848                 }
 849                 try {
 850                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 851                     br.delNeighbor(peerIp);
 852                 } catch (TException | BgpRouterException e) {
 853                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 854                 }
 855
 856                 final BgpAlarms bgpAlarms = getBgpAlarms();
 857                 if (bgpAlarms != null) {
 858                     bgpAlarms.clearBgpNbrDownAlarm(peerIp);
 859                 }
 860             }
 861         }
 862
 863         @Override
 864         protected void update(InstanceIdentifier<Neighbors> iid,
 865                 Neighbors oldval, Neighbors newval) {
 866             if (!isBGPEntityOwner()) {
 867                 return;
 868             }
 869             //purposefully nothing to do.
 870         }
 871     }
 872
 873     public class EbgpMultihopReactor
 874             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 875             implements ClusteredDataTreeChangeListener<EbgpMultihop> {
 876
 877         private static final String YANG_OBJ = "ebgp-multihop ";
 878
 879         public EbgpMultihopReactor() {
 880             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 881         }
 882
 883         @Override
 884         protected void add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 885             if (!isBGPEntityOwner()) {
 886                 return;
 887             }
 888             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 889             synchronized (BgpConfigurationManager.this) {
 890                 String peerIp = val.getPeerIp().getValue();
 891                 BgpRouter br = getClient(YANG_OBJ);
 892                 if (br == null) {
 893                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
 894                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 895                     return;
 896                 }
 897                 try {
 898                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 899                 } catch (TException | BgpRouterException e) {
 900                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 901                 }
 902             }
 903         }
 904
 905         @Override
 906         protected EbgpMultihopReactor getDataTreeChangeListener() {
 907             return EbgpMultihopReactor.this;
 908         }
 909
 910         @Override
 911         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
 912             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(EbgpMultihop.class);
 913         }
 914
 915         @Override
 916         protected void remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 917             if (!isBGPEntityOwner()) {
 918                 return;
 919             }
 920             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 921             synchronized (BgpConfigurationManager.this) {
 922                 String peerIp = val.getPeerIp().getValue();
 923                 BgpRouter br = getClient(YANG_OBJ);
 924                 if (br == null) {
 925                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 926                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 927                     return;
 928                 }
 929                 try {
 930                     br.delEbgpMultihop(peerIp);
 931                 } catch (TException | BgpRouterException e) {
 932                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 933                 }
 934             }
 935         }
 936
 937         @Override
 938         protected void update(InstanceIdentifier<EbgpMultihop> iid,
 939                 EbgpMultihop oldval, EbgpMultihop newval) {
 940             if (!isBGPEntityOwner()) {
 941                 return;
 942             }
 943             LOG.error(YANG_OBJ + UPD_WARN);
 944         }
 945     }
 946
 947     public class UpdateSourceReactor
 948             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
 949             implements ClusteredDataTreeChangeListener<UpdateSource> {
 950
 951         private static final String YANG_OBJ = "update-source ";
 952
 953         public UpdateSourceReactor() {
 954             super(UpdateSource.class, UpdateSourceReactor.class);
 955         }
 956
 957         @Override
 958         protected void add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 959             if (!isBGPEntityOwner()) {
 960                 return;
 961             }
 962             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
 963             synchronized (BgpConfigurationManager.this) {
 964                 String peerIp = val.getPeerIp().getValue();
 965                 BgpRouter br = getClient(YANG_OBJ);
 966                 if (br == null) {
 967                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
 968                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 969                     return;
 970                 }
 971                 try {
 972                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
 973                 } catch (TException | BgpRouterException e) {
 974                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 975                 }
 976             }
 977         }
 978
 979         @Override
 980         protected UpdateSourceReactor getDataTreeChangeListener() {
 981             return UpdateSourceReactor.this;
 982         }
 983
 984         @Override
 985         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
 986             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(UpdateSource.class);
 987         }
 988
 989         @Override
 990         protected void remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 991             if (!isBGPEntityOwner()) {
 992                 return;
 993             }
 994             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
 995             synchronized (BgpConfigurationManager.this) {
 996                 String peerIp = val.getPeerIp().getValue();
 997                 BgpRouter br = getClient(YANG_OBJ);
 998                 if (br == null) {
 999                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1000                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1001                     return;
1002                 }
1003                 try {
1004                     br.delUpdateSource(peerIp);
1005                 } catch (TException | BgpRouterException e) {
1006                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1007                 }
1008             }
1009         }
1010
1011         @Override
1012         protected void update(InstanceIdentifier<UpdateSource> iid,
1013                 UpdateSource oldval, UpdateSource newval) {
1014             if (!isBGPEntityOwner()) {
1015                 return;
1016             }
1017             LOG.error(YANG_OBJ + UPD_WARN);
1018         }
1019     }
1020
1021     public class AddressFamiliesReactor
1022             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
1023             implements ClusteredDataTreeChangeListener<AddressFamilies> {
1024
1025         private static final String YANG_OBJ = "address-families ";
1026
1027         public AddressFamiliesReactor() {
1028             super(AddressFamilies.class, AddressFamiliesReactor.class);
1029         }
1030
1031         @Override
1032         protected void add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1033             if (!isBGPEntityOwner()) {
1034                 return;
1035             }
1036             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
1037             synchronized (BgpConfigurationManager.this) {
1038                 String peerIp = val.getPeerIp().getValue();
1039                 BgpRouter br = getClient(YANG_OBJ);
1040                 if (br == null) {
1041                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1042                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1043                     return;
1044                 }
1045                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1046                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1047                 try {
1048                     br.addAddressFamily(peerIp, afi, safi);
1049                 } catch (TException | BgpRouterException e) {
1050                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1051                 }
1052             }
1053         }
1054
1055         @Override
1056         protected AddressFamiliesReactor getDataTreeChangeListener() {
1057             return AddressFamiliesReactor.this;
1058         }
1059
1060         @Override
1061         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
1062             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(AddressFamilies.class);
1063         }
1064
1065         @Override
1066         protected void remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1067             if (!isBGPEntityOwner()) {
1068                 return;
1069             }
1070             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
1071             synchronized (BgpConfigurationManager.this) {
1072                 String peerIp = val.getPeerIp().getValue();
1073                 BgpRouter br = getClient(YANG_OBJ);
1074                 if (br == null) {
1075                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1076                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1077                     return;
1078                 }
1079                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1080                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1081                 try {
1082                     br.delAddressFamily(peerIp, afi, safi);
1083                 } catch (TException | BgpRouterException e) {
1084                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1085                 }
1086             }
1087         }
1088
1089         @Override
1090         protected void update(InstanceIdentifier<AddressFamilies> iid,
1091                 AddressFamilies oldval, AddressFamilies newval) {
1092             if (!isBGPEntityOwner()) {
1093                 return;
1094             }
1095             LOG.error(YANG_OBJ + UPD_WARN);
1096         }
1097     }
1098
1099     public class NetworksReactor
1100             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
1101             implements ClusteredDataTreeChangeListener<Networks> {
1102
1103         private static final String YANG_OBJ = "networks ";
1104
1105         public NetworksReactor() {
1106             super(Networks.class, NetworksReactor.class);
1107         }
1108
1109         @Override
1110         public NetworksReactor getDataTreeChangeListener() {
1111             return NetworksReactor.this;
1112         }
1113
1114         @Override
1115         protected void add(InstanceIdentifier<Networks> iid, Networks val) {
1116             if (!isBGPEntityOwner()) {
1117                 return;
1118             }
1119             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1120             synchronized (BgpConfigurationManager.this) {
1121                 String rd = val.getRd();
1122                 String pfxlen = val.getPrefixLen();
1123                 String nh = val.getNexthop().getValue();
1124                 BgpRouter br = getClient(YANG_OBJ);
1125                 if (br == null) {
1126                     LOG.debug("{} Unable to process add for rd {} prefix {} nexthop {}; {} {}", YANG_OBJ, rd, pfxlen,
1127                             nh, BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1128                     return;
1129                 }
1130                 Long label = val.getLabel();
1131                 int lbl = label == null ? qbgpConstants.LBL_NO_LABEL
1132                         : label.intValue();
1133                 int l3vni = val.getL3vni() == null ? qbgpConstants.LBL_NO_LABEL
1134                         : val.getL3vni().intValue();
1135                 int l2vni = val.getL2vni() == null ? qbgpConstants.LBL_NO_LABEL
1136                         : val.getL2vni().intValue();
1137
1138                 BgpControlPlaneType protocolType = val.getBgpControlPlaneType();
1139                 int ethernetTag = val.getEthtag().intValue();
1140                 String esi = val.getEsi();
1141                 String macaddress = val.getMacaddress();
1142                 EncapType encapType = val.getEncapType();
1143                 String routerMac = val.getRoutermac();
1144
1145                 try {
1146                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni, BgpUtil.convertToThriftProtocolType(protocolType),
1147                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1148                 } catch (TException | BgpRouterException e) {
1149                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1150                 }
1151             }
1152         }
1153
1154         @Override
1155         protected InstanceIdentifier<Networks> getWildCardPath() {
1156             return InstanceIdentifier.create(Bgp.class).child(Networks.class);
1157         }
1158
1159         @Override
1160         protected void remove(InstanceIdentifier<Networks> iid, Networks val) {
1161             if (!isBGPEntityOwner()) {
1162                 return;
1163             }
1164             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1165             synchronized (BgpConfigurationManager.this) {
1166                 String rd = val.getRd();
1167                 String pfxlen = val.getPrefixLen();
1168                 BgpRouter br = getClient(YANG_OBJ);
1169                 if (br == null) {
1170                     LOG.debug("{} Unable to process remove for rd {} prefix {}; {} {}", YANG_OBJ, rd, pfxlen,
1171                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1172                     return;
1173                 }
1174                 Long label = val.getLabel();
1175                 int lbl = label == null ? 0 : label.intValue();
1176                 if (rd == null && lbl > 0) {
1177                     //LU prefix is being deleted.
1178                     rd = Integer.toString(lbl);
1179                 }
1180                 try {
1181                     br.delPrefix(rd, pfxlen);
1182                 } catch (TException | BgpRouterException e) {
1183                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1184                 }
1185             }
1186         }
1187
1188         /**get the value AFI from a prefix as "x.x.x.x/x".
1189          *
1190          * @param pfxlen the prefix to get an afi
1191          * @return the afi value as you are need
1192          */
1193         public  int  testValueAFI(String pfxlen) {
1194             int afiNew = af_afi.AFI_IP.getValue();
1195             try {
1196                 String ipOnly = pfxlen.substring(0, pfxlen.lastIndexOf("/"));
1197                 java.net.Inet6Address.getByName(ipOnly);
1198                 afiNew = af_afi.AFI_IPV6.getValue();
1199             } catch (java.net.UnknownHostException e) {
1200                 //ce n'est pas de l'ipv6
1201             }
1202             return afiNew;
1203         }
1204
1205
1206         @Override
1207         protected void update(final InstanceIdentifier<Networks> iid,
1208                 final Networks oldval, final Networks newval) {
1209             if (!isBGPEntityOwner()) {
1210                 return;
1211             }
1212             if (oldval.equals(newval)) {
1213                 //Update: OLD and New values are same, no need to trigger remove/add.
1214                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1215                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1216                 return;
1217             }
1218             LOG.debug("update networks old val RD: {}, Prefix: {}, Label: {}, NH: {} "
1219                             + "new val RD: {}, Prefix: {}, Label: {}, NH: {}",
1220                     oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop(),
1221                     newval.getRd(), newval.getPrefixLen(), newval.getLabel(), newval.getNexthop());
1222             remove(iid, oldval);
1223             add(iid, newval);
1224         }
1225     }
1226
1227     static Timer timer = new Timer();
1228
1229     public class VrfsReactor
1230             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1231             implements ClusteredDataTreeChangeListener<Vrfs> {
1232
1233         private static final String YANG_OBJ = "vrfs ";
1234
1235         public VrfsReactor() {
1236             super(Vrfs.class, VrfsReactor.class);
1237         }
1238
1239         @Override
1240         protected void add(InstanceIdentifier<Vrfs> iid, Vrfs vrfs) {
1241             if (!isBGPEntityOwner()) {
1242                 return;
1243             }
1244             LOG.debug("received add Vrfs config value {}", vrfs.getRd());
1245             synchronized (BgpConfigurationManager.this) {
1246                 String rd = vrfs.getRd();
1247                 BgpRouter br = getClient(YANG_OBJ);
1248                 if (br == null) {
1249                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1250                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1251                     return;
1252                 }
1253                 try {
1254                     List<AddressFamiliesVrf> vrfAddrFamilyList = vrfs.getAddressFamiliesVrf();
1255                     for (AddressFamiliesVrf vrfAddrFamily : vrfAddrFamilyList) {
1256                         /*add to br the new vrfs arguments*/
1257                         br.addVrf(BgpUtil.getLayerType(vrfAddrFamily), rd, vrfs.getImportRts(), vrfs.getExportRts()
1258                         );
1259                     }
1260                     /*add to br the vrfs contained in mapNewAdFamily*/
1261                     List<AddressFamiliesVrf> vrfAddrFamilyListFromMap = mapNewAdFamily.get(rd);
1262                     if (vrfAddrFamilyListFromMap == null) {
1263                         return;
1264                     }
1265
1266                     for (AddressFamiliesVrf adf : vrfAddrFamilyListFromMap) {
1267                         if (vrfAddrFamilyList.contains(adf)) {
1268                             mapNewAdFamily.remove(rd);
1269                         } else  if (adf != null) {
1270
1271                             br.addVrf(BgpUtil.getLayerType(adf), rd, vrfs.getImportRts(), vrfs.getExportRts()
1272                             );
1273                             // remove AddressFamiliesVrf which was already added to BGP
1274                             vrfAddrFamilyListFromMap.remove(adf);
1275                             if (vrfAddrFamilyListFromMap.isEmpty()) {
1276                                 // remove Vrf entry from temp mapNewAdFamily if all its AddressFamiliesVrf was
1277                                 // added to BGP
1278                                 mapNewAdFamily.remove(rd);
1279                             }
1280                         }
1281                     }
1282                 } catch (TException | BgpRouterException e) {
1283                     LOG.error("{} get {}, Add received exception", YANG_OBJ, ADD_WARN, e);
1284                 }
1285             }
1286         }
1287
1288         @Override
1289         protected VrfsReactor getDataTreeChangeListener() {
1290             return VrfsReactor.this;
1291         }
1292
1293         @Override
1294         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1295             return InstanceIdentifier.create(Bgp.class).child(Vrfs.class);
1296         }
1297
1298         @Override
1299         protected void remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1300             if (!isBGPEntityOwner()) {
1301                 return;
1302             }
1303             LOG.debug("received remove Vrfs config val {}", val.getRd());
1304             synchronized (BgpConfigurationManager.this) {
1305                 String rd = val.getRd();
1306                 BgpRouter br = getClient(YANG_OBJ);
1307                 if (br == null) {
1308                     LOG.debug("{} Unable to process remove for rd {}; {} {}", YANG_OBJ, rd,
1309                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1310                     return;
1311                 }
1312                 try {
1313                     List<AddressFamiliesVrf> adf = mapNewAdFamily.get(rd);
1314                     adf = adf != null ? adf : new ArrayList<>();
1315                     for (AddressFamiliesVrf s : val.getAddressFamiliesVrf()) {
1316                         br.delVrf(rd, s.getAfi(), s.getSafi());
1317                         adf.remove(s);// remove in the map the vrf in waiting for advertise quagga
1318                     }
1319                     if (adf.isEmpty()) {
1320                         mapNewAdFamily.remove(rd);
1321                     }
1322                 } catch (TException | BgpRouterException e) {
1323                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1324                 }
1325             }
1326         }
1327
1328         @Override
1329         protected void update(InstanceIdentifier<Vrfs> iid,
1330                 Vrfs oldval, Vrfs newval) {
1331             if (oldval != null && newval != null) {
1332                 LOG.debug("received update Vrfs config val {}, VRFS: Update getting triggered for VRFS rd {}",
1333                         newval.getRd(), oldval.getRd());
1334             } else {
1335                 LOG.debug("received update Vrfs config val {}, from old vrf {}",
1336                         newval, oldval);
1337             }
1338             if (!isBGPEntityOwner()) {
1339                 return;
1340             }
1341
1342             List<AddressFamiliesVrf> adFamilyVrfToDel = new ArrayList<>();
1343             List<AddressFamiliesVrf> adFamilyVrfToAdd = new ArrayList<>();
1344             List<AddressFamiliesVrf> oldlistAdFamilies = new ArrayList<>();
1345             List<AddressFamiliesVrf> newlistAdFamilies = new ArrayList<>();
1346             if (oldval != null) {
1347                 oldlistAdFamilies = oldval.getAddressFamiliesVrf() == null
1348                         ? new ArrayList<>() : oldval.getAddressFamiliesVrf();
1349             }
1350             if (newval != null) {
1351                 newlistAdFamilies = newval.getAddressFamiliesVrf() == null
1352                         ? new ArrayList<>() : newval.getAddressFamiliesVrf();
1353             }
1354             /*find old AddressFamily to remove from new configuration*/
1355             for (AddressFamiliesVrf adVrf : oldlistAdFamilies) {
1356                 if (!newlistAdFamilies.contains(adVrf)) {
1357                     adFamilyVrfToDel.add(adVrf);
1358                 }
1359             }
1360             /*find new AddressFamily to add to unexisting configuration*/
1361             for (AddressFamiliesVrf adVrf : newlistAdFamilies) {
1362                 if (!oldlistAdFamilies.contains(adVrf)) {
1363                     adFamilyVrfToAdd.add(adVrf);
1364                 }
1365             }
1366             String rd = newval != null ? newval.getRd() : null;
1367             if (rd != null) {
1368                 BgpRouter br = getClient(YANG_OBJ);
1369                 if (br == null) {
1370                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1371                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1372                     return;
1373                 }
1374
1375                 for (AddressFamiliesVrf adfvrf : adFamilyVrfToAdd) {
1376                     try {
1377                         LOG.debug("call addVRf rd {} afi {} safi {}", rd, adfvrf.getAfi(), adfvrf.getSafi());
1378                         br.addVrf(BgpUtil.getLayerType(adfvrf), rd, newval.getImportRts(),
1379                                 newval.getExportRts());
1380                     } catch (TException | BgpRouterException e) {
1381                         LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1382                     }
1383                 }
1384
1385                 for (AddressFamiliesVrf adfToDel : adFamilyVrfToDel) {
1386                     try {
1387                         LOG.debug("call delVRf rd {} afi {} safi {}", rd, adfToDel.getAfi(), adfToDel.getSafi());
1388                         br.delVrf(rd, adfToDel.getAfi(), adfToDel.getSafi());
1389                     } catch (TException | BgpRouterException e) {
1390                         LOG.error("{} delVrf received exception; {}", YANG_OBJ, ADD_WARN, e);
1391                     }
1392                 }
1393             }
1394         }
1395     }
1396
1397     protected void activateMIP() {
1398         try {
1399             LOG.trace("BgpReactor: Executing MIP Activate command");
1400             Runtime.getRuntime().exec("cluster ip -a sdnc_bgp_mip");
1401             Runtime.getRuntime().exec("cluster ip -a sdnc_os_mip");
1402             LOG.trace("bgpMIP Activated");
1403
1404         } catch (IOException io) {
1405             LOG.error("IO Exception got while activating mip: {}", io.getMessage());
1406         }
1407     }
1408
1409     public class BgpReactor
1410             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1411             implements ClusteredDataTreeChangeListener<Bgp> {
1412
1413         private static final String YANG_OBJ = "Bgp ";
1414
1415         public BgpReactor() {
1416             super(Bgp.class, BgpReactor.class);
1417         }
1418
1419
1420         @Override
1421         protected void add(InstanceIdentifier<Bgp> iid, Bgp val) {
1422             LOG.debug("received add Bgp config");
1423
1424             try {
1425                 initer.await();
1426             } catch (InterruptedException e) {
1427                 // Ignored
1428             }
1429             synchronized (BgpConfigurationManager.this) {
1430                 config = val;
1431                 if (!isBGPEntityOwner()) {
1432                     return;
1433                 }
1434                 activateMIP();
1435             }
1436         }
1437
1438         @Override
1439         protected BgpReactor getDataTreeChangeListener() {
1440             return BgpReactor.this;
1441         }
1442
1443         @Override
1444         protected InstanceIdentifier<Bgp> getWildCardPath() {
1445             return InstanceIdentifier.create(Bgp.class);
1446         }
1447
1448         @Override
1449         protected void remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1450             if (!isBGPEntityOwner()) {
1451                 return;
1452             }
1453             LOG.debug("received remove Bgp config");
1454
1455             config = null;
1456         }
1457
1458         @Override
1459         protected void update(InstanceIdentifier<Bgp> iid,
1460                 Bgp oldval, Bgp newval) {
1461             if (!isBGPEntityOwner()) {
1462                 return;
1463             }
1464
1465             config = newval;
1466         }
1467     }
1468
1469     @SuppressWarnings("deprecation")
1470     public class MultipathReactor
1471             extends AsyncDataTreeChangeListenerBase<Multipath, MultipathReactor>
1472             implements ClusteredDataTreeChangeListener<Multipath> {
1473
1474         private static final String YANG_OBJ = "multipath ";
1475
1476         public MultipathReactor() {
1477             super(Multipath.class, MultipathReactor.class);
1478         }
1479
1480
1481         @Override
1482         protected MultipathReactor getDataTreeChangeListener() {
1483             return MultipathReactor.this;
1484         }
1485
1486         @Override
1487         protected InstanceIdentifier<Multipath> getWildCardPath() {
1488             return InstanceIdentifier.create(Bgp.class).child(Multipath.class);
1489         }
1490
1491         @Override
1492         protected void remove(InstanceIdentifier<Multipath> iid, Multipath val) {
1493             executor.execute(new MultipathStatusChange(val));
1494         }
1495
1496         @Override
1497         protected void update(InstanceIdentifier<Multipath> iid, Multipath oldval, Multipath newval) {
1498             executor.execute(new MultipathStatusChange(newval));
1499         }
1500
1501         @Override
1502         protected void add(InstanceIdentifier<Multipath> key, Multipath dataObjectModification) {
1503             executor.execute(new MultipathStatusChange(dataObjectModification));
1504         }
1505
1506         class MultipathStatusChange implements Runnable {
1507
1508             Multipath multipath;
1509
1510             MultipathStatusChange(Multipath multipath) {
1511                 this.multipath = multipath;
1512             }
1513
1514             @Override
1515             public void run() {
1516                 if (isBGPEntityOwner()) {
1517                     synchronized (BgpConfigurationManager.this) {
1518
1519                         BgpRouter br = getClient(YANG_OBJ);
1520
1521                         if (br != null) {
1522                             af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1523                             af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1524
1525                             try {
1526                                 if (multipath.isMultipathEnabled()) {
1527                                     br.enableMultipath(afi, safi);
1528                                 } else {
1529                                     br.disableMultipath(afi, safi);
1530                                 }
1531                             } catch (TException | BgpRouterException e) {
1532                                 LOG.error("{} received exception", YANG_OBJ, e);
1533                             }
1534                         }
1535                     }
1536                 }
1537             }
1538
1539         }
1540
1541         @Override
1542         public void close() {
1543             super.close();
1544         }
1545     }
1546
1547     @SuppressWarnings("deprecation")
1548     public class VrfMaxpathReactor
1549             extends AsyncDataTreeChangeListenerBase<VrfMaxpath, VrfMaxpathReactor>
1550             implements ClusteredDataTreeChangeListener<VrfMaxpath> {
1551
1552         private static final String YANG_OBJ = "vrfMaxpath ";
1553
1554         public VrfMaxpathReactor() {
1555             super(VrfMaxpath.class, VrfMaxpathReactor.class);
1556         }
1557
1558
1559         @Override
1560         protected VrfMaxpathReactor getDataTreeChangeListener() {
1561             return VrfMaxpathReactor.this;
1562         }
1563
1564         @Override
1565         protected InstanceIdentifier<VrfMaxpath> getWildCardPath() {
1566             return InstanceIdentifier.create(Bgp.class).child(VrfMaxpath.class);
1567         }
1568
1569         class VrfMaxPathConfigurator implements Runnable {
1570
1571             VrfMaxpath vrfMaxpathVal;
1572
1573             VrfMaxPathConfigurator(VrfMaxpath vrfMaxPathVal) {
1574                 this.vrfMaxpathVal = vrfMaxPathVal;
1575             }
1576
1577             @Override
1578             public void run() {
1579                 if (isBGPEntityOwner()) {
1580                     synchronized (BgpConfigurationManager.this) {
1581                         BgpRouter br = getClient(YANG_OBJ);
1582                         if (br != null) {
1583                             try {
1584                                 br.multipaths(vrfMaxpathVal.getRd(), vrfMaxpathVal.getMaxpaths());
1585                                 LOG.debug("Maxpath for vrf {} is {}", vrfMaxpathVal.getRd(),
1586                                         vrfMaxpathVal.getMaxpaths());
1587                             } catch (TException | BgpRouterException e) {
1588                                 LOG.error("{} received exception", YANG_OBJ, e);
1589                             }
1590                         }
1591                     }
1592                 }
1593             }
1594         }
1595
1596         @Override
1597         protected void remove(InstanceIdentifier<VrfMaxpath> iid, VrfMaxpath vrfMaxPathVal) {
1598             executor.execute(new VrfMaxPathConfigurator(vrfMaxPathVal));
1599         }
1600
1601         @Override
1602         protected void update(InstanceIdentifier<VrfMaxpath> iid,
1603                               VrfMaxpath oldval, VrfMaxpath newval) {
1604             if (!Objects.equals(oldval.getMaxpaths(), newval.getMaxpaths())) {
1605                 executor.execute(new VrfMaxPathConfigurator(newval));
1606             }
1607         }
1608
1609         @Override
1610         protected void add(InstanceIdentifier<VrfMaxpath> instanceIdentifier, VrfMaxpath vrfMaxpathVal) {
1611             executor.execute(new VrfMaxPathConfigurator(vrfMaxpathVal));
1612         }
1613
1614         @Override
1615         public void close() {
1616             super.close();
1617         }
1618     }
1619
1620     public boolean isIpAvailable(String odlip) {
1621
1622         try {
1623             if (odlip != null) {
1624                 if ("127.0.0.1".equals(odlip)) {
1625                     return true;
1626                 }
1627                 Enumeration<NetworkInterface> networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces();
1628                 while (networkInterfaceEnumeration.hasMoreElements()) {
1629                     NetworkInterface networkInterface = networkInterfaceEnumeration.nextElement();
1630                     Enumeration<InetAddress>  inetAddressEnumeration = networkInterface.getInetAddresses();
1631                     while (inetAddressEnumeration.hasMoreElements()) {
1632                         InetAddress inetAddress = inetAddressEnumeration.nextElement();
1633                         if (odlip.equals(inetAddress.getHostAddress())) {
1634                             return true;
1635                         }
1636                     }
1637                 }
1638             }
1639         } catch (SocketException e) {
1640             // Ignored?
1641         }
1642         return false;
1643     }
1644
1645     public long getStalePathtime(int defValue, AsId asId) {
1646         long spt = 0;
1647         try {
1648             spt = getConfig().getGracefulRestart().getStalepathTime();
1649         } catch (NullPointerException e) {
1650             try {
1651                 spt = asId.getStalepathTime();
1652                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1653             } catch (NullPointerException ignore) {
1654                 LOG.trace("BGP AS id is not set using graceful");
1655                 spt = defValue;
1656             }
1657         }
1658         if (spt == 0) {
1659             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1660             spt = defValue;
1661         }
1662         return spt;
1663     }
1664
1665     public static boolean isValidConfigBgpHostPort(String bgpHost, int bgpPort) {
1666         if (!bgpHost.equals(DEF_CHOST)) {
1667             return true;
1668         } else {
1669             return false;
1670         }
1671     }
1672
1673     public synchronized void bgpRestarted() {
1674         /*
1675          * If there a thread which in the process of stale cleanup, cancel it
1676          * and start a new thread (to avoid processing same again).
1677          */
1678         if (previousReplayJobInProgress()) {
1679             cancelPreviousReplayJob();
1680         }
1681         Runnable task = () -> {
1682             try {
1683                 LOG.info("running bgp replay task ");
1684                 if (get() == null) {
1685                     String host = getConfigHost();
1686                     int port = getConfigPort();
1687                     LOG.info("connecting  to bgp host {} ", host);
1688                     bgpRouter.connect(host, port);
1689                     LOG.info("no config to push in bgp replay task ");
1690                     return;
1691                 }
1692                 setStaleStartTime(System.currentTimeMillis());
1693                 LOG.info("started creating stale fibDSWriter  map ");
1694                 createStaleFibMap();
1695                 setStaleEndTime(System.currentTimeMillis());
1696                 LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
1697                 LOG.info("started bgp config replay ");
1698                 setCfgReplayStartTime(System.currentTimeMillis());
1699                 boolean replaySucceded = replay();
1700                 setCfgReplayEndTime(System.currentTimeMillis());
1701                 LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1702                 if (replaySucceded) {
1703                     LOG.info("starting the stale cleanup timer");
1704                     long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1705                     setStaleCleanupTime(routeSyncTime);
1706                     routeCleanupFuture = executor.schedule(new RouteCleanup(), routeSyncTime, TimeUnit.SECONDS);
1707                 } else {
1708                     staledFibEntriesMap.clear();
1709                 }
1710             } catch (InterruptedException | TimeoutException | ExecutionException eCancel) {
1711                 LOG.error("Stale Cleanup Task Cancelled", eCancel);
1712             }
1713         };
1714         lastReplayJobFt = executor.submit(task);
1715     }
1716
1717     private boolean previousReplayJobInProgress() {
1718         return lastReplayJobFt != null && !lastReplayJobFt.isDone();
1719     }
1720
1721     private void cancelPreviousReplayJob() {
1722         try {
1723             LOG.error("cancelling already running bgp replay task");
1724             if (lastReplayJobFt != null) {
1725                 lastReplayJobFt.cancel(true);
1726                 lastReplayJobFt = null;
1727                 staledFibEntriesMap.clear();
1728             }
1729             if (routeCleanupFuture != null) {
1730                 routeCleanupFuture.cancel(true);
1731                 routeCleanupFuture = null;
1732                 staledFibEntriesMap.clear();
1733             }
1734             Thread.sleep(2000);
1735         } catch (InterruptedException e) {
1736             LOG.error("Failed to cancel previous replay job ", e);
1737         }
1738     }
1739
1740     private void doRouteSync() {
1741         LOG.error("Starting BGP route sync");
1742         try {
1743             bgpRouter.initRibSync(bgpSyncHandle);
1744         } catch (BgpRouterException e) {
1745             LOG.error("Route sync aborted, exception when initializing", e);
1746             return;
1747         }
1748         while (bgpSyncHandle.getState() != BgpSyncHandle.DONE) {
1749             for (af_afi afi : af_afi.values()) {
1750                 Routes routes = null;
1751                 try {
1752                     routes = bgpRouter.doRibSync(bgpSyncHandle, afi);
1753                 } catch (TException | BgpRouterException e) {
1754                     LOG.error("Route sync aborted, exception when syncing", e);
1755                     return;
1756                 }
1757                 Iterator<Update> updates = routes.getUpdatesIterator();
1758                 while (updates.hasNext()) {
1759                     Update update = updates.next();
1760                     String rd = update.getRd();
1761                     String nexthop = update.getNexthop();
1762
1763                     // TODO: decide correct label here
1764                     int label = update.getL3label();
1765                     int l2label = update.getL2label();
1766
1767                     String prefix = update.getPrefix();
1768                     int plen = update.getPrefixlen();
1769
1770
1771                     // TODO: protocol type will not be available in "update"
1772                     // use "rd" to query vrf table and obtain the protocol_type.
1773                     // Currently using PROTOCOL_EVPN as default.
1774                     onUpdatePushRoute(
1775                            protocol_type.PROTOCOL_EVPN,
1776                            rd,
1777                            prefix,
1778                            plen,
1779                            nexthop,
1780                            update.getMacaddress(),
1781                            label,
1782                            l2label,
1783                            update.getRoutermac(),
1784                            afi);
1785                 }
1786             }
1787         }
1788         try {
1789             LOG.error("Ending BGP route-sync");
1790             bgpRouter.endRibSync(bgpSyncHandle);
1791         } catch (BgpRouterException e) {
1792             // Ignored?
1793         }
1794     }
1795
1796     public void addTepToElanDS(String rd, String tepIp, String mac, Long l2vni) {
1797         boolean needUpdate = addToRt2TepMap(rd, tepIp, mac, l2vni);
1798         if (needUpdate) {
1799             LOG.info("Adding tepIp {} with RD {} to ELan DS", tepIp, rd);
1800             bgpUtil.addTepToElanInstance(rd, tepIp);
1801         } else {
1802             LOG.debug("Skipping the Elan update for RT2 from tep {} rd {}", tepIp, rd);
1803         }
1804     }
1805
1806     public void deleteTepfromElanDS(String rd, String tepIp, String mac) {
1807         boolean needUpdate = deleteFromRt2TepMap(rd, tepIp, mac);
1808         if (needUpdate) {
1809             LOG.info("Deleting tepIp {} with RD {} to ELan DS", tepIp, rd);
1810             bgpUtil.deleteTepFromElanInstance(rd, tepIp);
1811         } else {
1812             LOG.debug("Skipping the Elan update for RT2 withdraw from tep {} rd {}", tepIp, rd);
1813         }
1814     }
1815
1816     /* onUpdatePushRoute
1817      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1818      *  - Entry compare shall include NextHop, Label.
1819      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1820      *  - If entry not found, add to FIB Config DS.
1821      *  - If entry found, but either Label/NextHop doesn't match.
1822      *      - Update FIB Config DS with modified values.
1823      *      - delete from Stale Map.
1824      */
1825
1826     public void onUpdatePushRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
1827                                   String macaddress, int label, int l2label, String routermac, af_afi afi) {
1828         boolean addroute = false;
1829         boolean macupdate = false;
1830         long l3vni = 0L;
1831         VrfEntry.EncapType encapType = VrfEntry.EncapType.Mplsgre;
1832         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1833             encapType = VrfEntry.EncapType.Vxlan;
1834             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
1835             if (vpnInstanceOpDataEntry != null) {
1836                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
1837                     LOG.info("Got RT2 route for RD {} l3label {} l2label {} from tep {} with mac {} remote RD {}",
1838                             vpnInstanceOpDataEntry.getVpnInstanceName(), label, l2label, nextHop, macaddress, rd);
1839                     addTepToElanDS(rd, nextHop, macaddress, (long)l2label);
1840                     macupdate = true;
1841                 } else {
1842                     l3vni = vpnInstanceOpDataEntry.getL3vni();
1843                 }
1844             } else {
1845                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1846                 return;
1847             }
1848         }
1849
1850         if (!staledFibEntriesMap.isEmpty()) {
1851             // restart Scenario, as MAP is not empty.
1852             Map<String, Long> map = staledFibEntriesMap.get(rd);
1853             if (map != null) {
1854                 String prefixNextHop = appendNextHopToPrefix(prefix + "/" + plen, nextHop);
1855                 Long labelInStaleMap = map.get(prefixNextHop);
1856                 if (null == labelInStaleMap) {
1857                     // New Entry, which happened to be added during restart.
1858                     addroute = true;
1859                 } else {
1860                     map.remove(prefixNextHop);
1861                     if (isRouteModified(label, labelInStaleMap)) {
1862                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1863                         // Existing entry, where in Label got modified during restart
1864                         addroute = true;
1865                     }
1866                 }
1867             } else {
1868                 LOG.debug("rd {} map is null while processing prefix {} ", rd, prefix);
1869                 addroute = true;
1870             }
1871         } else {
1872             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1873             addroute = true;
1874         }
1875         if (macupdate) {
1876             LOG.info("ADD: Adding Mac Fib entry rd {} mac{} nexthop {} l2vni {}", rd, macaddress, nextHop, l2label);
1877             fibDSWriter.addMacEntryToDS(rd, macaddress, prefix, Collections.singletonList(nextHop),
1878                     encapType, l2label, routermac, RouteOrigin.BGP);
1879             LOG.info("ADD: Added Mac Fib entry rd {} prefix {} nexthop {} label {}", rd, macaddress, nextHop, l2label);
1880         } else if (addroute) {
1881             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {} afi {}",
1882                     rd, prefix, nextHop, label, afi);
1883             // TODO: modify addFibEntryToDS signature
1884             List<String> nextHopList = Collections.singletonList(nextHop);
1885             fibDSWriter.addFibEntryToDS(rd, prefix + "/" + plen, nextHopList, encapType, label, l3vni,
1886                                         routermac, RouteOrigin.BGP);
1887             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1888             String vpnName = bgpUtil.getVpnNameFromRd(rd);
1889             if (vpnName != null) {
1890                 vpnLinkService.leakRouteIfNeeded(vpnName, prefix, nextHopList, label, RouteOrigin.BGP,
1891                                                  NwConstants.ADD_FLOW);
1892             }
1893         }
1894     }
1895
1896     public void onUpdateWithdrawRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
1897             String macaddress) {
1898         long vni = 0L;
1899         boolean macupdate = false;
1900         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1901             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
1902             if (vpnInstanceOpDataEntry != null) {
1903                 vni = vpnInstanceOpDataEntry.getL3vni();
1904                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
1905                     LOG.debug("Got RT2 withdraw for RD {} {} from tep {} with mac {} remote RD {}",
1906                             vpnInstanceOpDataEntry.getVpnInstanceName(), vni, nextHop, macaddress, rd);
1907                     deleteTepfromElanDS(rd, nextHop, macaddress);
1908                     LOG.debug("For rd {}. skipping fib update", rd);
1909                     macupdate = true;
1910                 }
1911             } else {
1912                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1913                 return;
1914             }
1915         }
1916         if (macupdate) {
1917             LOG.info("Removing Mac Fib entry rd {} mac{} nexthop {} ", rd, macaddress, nextHop);
1918             fibDSWriter.removeMacEntryFromDS(rd, macaddress);
1919             LOG.info("Removed Mac Fib entry rd {} prefix {} nexthop {} ", rd, macaddress, nextHop);
1920         } else {
1921             LOG.info("REMOVE: Removing Fib entry rd {} prefix {}", rd, prefix);
1922             fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nextHop);
1923             LOG.info("REMOVE: Removed Fib entry rd {} prefix {}", rd, prefix);
1924         }
1925     }
1926
1927     //TODO: below function is for testing purpose with cli
1928     public void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
1929         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
1930         fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nexthop);
1931         String vpnName = bgpUtil.getVpnNameFromRd(rd);
1932         if (vpnName != null) {
1933             vpnLinkService.leakRouteIfNeeded(vpnName, prefix, null /*nextHopList*/, 0 /*INVALID_LABEL*/,
1934                                              RouteOrigin.BGP, NwConstants.DEL_FLOW);
1935         }
1936     }
1937
1938     private static boolean isRouteModified(int label, Long labelInStaleMap) {
1939         return labelInStaleMap != null && !labelInStaleMap.equals(Long.valueOf(label));
1940     }
1941
1942     static class ReplayNbr {
1943         Neighbors nbr;
1944         boolean shouldRetry = false;
1945
1946         public Neighbors getNbr() {
1947             return nbr;
1948         }
1949
1950         public boolean isShouldRetry() {
1951             return shouldRetry;
1952         }
1953
1954         public void setShouldRetry(boolean retryNbr) {
1955             this.shouldRetry = retryNbr;
1956         }
1957
1958         ReplayNbr(Neighbors nbr, boolean shouldRetry) {
1959             this.nbr = nbr;
1960             this.shouldRetry = shouldRetry;
1961         }
1962     }
1963
1964     private static boolean replayNbrConfig(List<Neighbors> neighbors, BgpRouter br) {
1965         if (neighbors == null || neighbors.isEmpty()) {
1966             LOG.error("Replaying nbr configuration, received NULL list ");
1967             return true;
1968         }
1969
1970         List<ReplayNbr> replayNbrList = new ArrayList<>();
1971         for (Neighbors nbr : neighbors) {
1972             if (nbr != null) {
1973                 replayNbrList.add(new ReplayNbr(nbr, true));
1974             }
1975         }
1976         final int numberOfNbrRetries = 3;
1977         RetryOnException nbrRetry = new RetryOnException(numberOfNbrRetries);
1978         do {
1979             for (ReplayNbr replayNbr : replayNbrList) {
1980                 if (!replayNbr.isShouldRetry()) {
1981                     continue;
1982                 }
1983                 boolean replayDone = false;
1984                 LOG.debug("Replaying addNbr {}", replayNbr.getNbr().getAddress().getValue());
1985                 replayDone = false;
1986                 try {
1987                     final String md5password = extractMd5Secret(replayNbr.getNbr());
1988                     br.addNeighbor(replayNbr.getNbr().getAddress().getValue(),
1989                             replayNbr.getNbr().getRemoteAs().longValue(), md5password);
1990                     UpdateSource us = replayNbr.getNbr().getUpdateSource();
1991                     if (us != null) {
1992                         LOG.debug("Replaying updatesource along with nbr: {} US-ip: {} to peer {}",
1993                                 replayNbr.getNbr().getAddress().getValue(),
1994                                 us.getSourceIp().getValue(),
1995                                 us.getPeerIp().getValue());
1996                         br.addUpdateSource(us.getPeerIp().getValue(),
1997                                 us.getSourceIp().getValue());
1998                     }
1999                     replayDone = true;
2000                 } catch (TException | BgpRouterException eNbr) {
2001                     LOG.debug("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), eNbr);
2002                 }
2003                 boolean replaySuccess = true;
2004                 replaySuccess = replaySuccess && replayDone;
2005                 LOG.debug("Replay addNbr {} successful", replayNbr.getNbr().getAddress().getValue());
2006
2007                 //Update Source handling
2008                 UpdateSource us = replayNbr.getNbr().getUpdateSource();
2009                 if (replayDone == false && us != null) {
2010                     LOG.debug("Replaying updatesource {} to peer {}", us.getSourceIp().getValue(),
2011                             us.getPeerIp().getValue());
2012                     replayDone = false;
2013                     try {
2014                         br.addUpdateSource(us.getPeerIp().getValue(),
2015                                 us.getSourceIp().getValue());
2016                         replayDone = true;
2017                     } catch (TException | BgpRouterException eUs) {
2018                         LOG.debug("Replaying UpdateSource for Nbr {}, exception:",
2019                                 replayNbr.getNbr().getAddress().getValue(), eUs);
2020                     }
2021                     LOG.debug("Replay updatesource {} successful", us.getSourceIp().getValue());
2022                     replaySuccess = replaySuccess && replayDone;
2023                 }
2024                 //Ebgp Multihope
2025                 EbgpMultihop en = replayNbr.getNbr().getEbgpMultihop();
2026                 if (en != null) {
2027                     replayDone = false;
2028                     try {
2029                         br.addEbgpMultihop(en.getPeerIp().getValue(),
2030                                 en.getNhops().intValue());
2031                         replayDone = true;
2032                     } catch (TException | BgpRouterException eEbgpMhop) {
2033                         LOG.debug("Replaying EbgpMultihop for Nbr {}, exception: ",
2034                                 replayNbr.getNbr().getAddress().getValue(), eEbgpMhop);
2035                     }
2036                     replaySuccess = replaySuccess && replayDone;
2037                 }
2038
2039                 //afs
2040                 List<AddressFamilies> afs = replayNbr.getNbr().getAddressFamilies();
2041                 if (afs != null) {
2042                     for (AddressFamilies af : afs) {
2043                         af_afi afi = af_afi.findByValue(af.getAfi().intValue());
2044                         af_safi safi = af_safi.findByValue(af.getSafi().intValue());
2045                         replayDone = false;
2046                         try {
2047                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
2048                             replayDone = true;
2049                         } catch (TException | BgpRouterException eAFs) {
2050                             LOG.debug("Replaying AddressFamily for Nbr {}, exception:",
2051                                     replayNbr.getNbr().getAddress().getValue(), eAFs);
2052                         }
2053                         replaySuccess = replaySuccess && replayDone;
2054                     }
2055                 }
2056                 //replay is success --> no need to replay this nbr in next iteration.
2057                 replayNbr.setShouldRetry(replaySuccess ? false : true);
2058             }
2059         } while (nbrRetry.decrementAndRetry());
2060         boolean replaySuccess = true;
2061         for (ReplayNbr replayNbr : replayNbrList) {
2062             replaySuccess = replaySuccess && !replayNbr.isShouldRetry();
2063         }
2064         return replaySuccess;
2065     }
2066
2067     public String getConfigHost() {
2068         if (config == null) {
2069             return hostStartup;
2070         }
2071         ConfigServer ts = config.getConfigServer();
2072         return ts == null ? hostStartup : ts.getHost().getValue();
2073     }
2074
2075     public int getConfigPort() {
2076         if (config == null) {
2077             return Integer.parseInt(portStartup);
2078         }
2079         ConfigServer ts = config.getConfigServer();
2080         return ts == null ? Integer.parseInt(portStartup) :
2081                 ts.getPort().intValue();
2082     }
2083
2084     public Bgp getConfig() {
2085         AtomicInteger bgpDSretryCount = new AtomicInteger(DS_RETRY_COUNT);
2086         while (0 != bgpDSretryCount.decrementAndGet()) {
2087             try {
2088                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
2089                         InstanceIdentifier.create(Bgp.class)).orNull();
2090             } catch (ReadFailedException e) {
2091                 //Config DS may not be up, so sleep for 1 second and retry
2092                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
2093                 try {
2094                     Thread.sleep(WAIT_TIME_BETWEEN_EACH_TRY_MILLIS);
2095                 } catch (InterruptedException timerEx) {
2096                     LOG.debug("WAIT_TIME_BETWEEN_EACH_TRY_MILLIS, Timer got interrupted while waiting for"
2097                             + "config DS availability", timerEx);
2098                 }
2099             }
2100         }
2101         LOG.error("failed to get bgp config");
2102         return null;
2103     }
2104
2105     @SuppressWarnings("checkstyle:IllegalCatch")
2106     public synchronized boolean replay() throws InterruptedException, TimeoutException, ExecutionException {
2107         boolean replaySucceded = true;
2108         String host = getConfigHost();
2109         int port = getConfigPort();
2110         LOG.error("connecting  to bgp host {} ", host);
2111         boolean res = bgpRouter.connect(host, port);
2112         if (!res) {
2113             LOG.error("Cannot connect to BGP config server at {}:{}{}", host, port,
2114                     config != null ? "; Configuration Replay aborted" : "");
2115             return replaySucceded;
2116         }
2117         config = getConfig();
2118         if (config == null) {
2119             LOG.error("bgp config is empty nothing to push to bgp");
2120             return replaySucceded;
2121         }
2122         BgpRouter br = bgpRouter;
2123         AsId asId = config.getAsId();
2124         if (asId == null) {
2125             LOG.error("bgp as-id is null");
2126             return replaySucceded;
2127         }
2128         long asNum = asId.getLocalAs();
2129         IpAddress routerId = asId.getRouterId();
2130         String rid = routerId == null ? "" : routerId.stringValue();
2131         int stalepathTime = (int) getStalePathtime(RESTART_DEFAULT_GR, config.getAsId());
2132         boolean announceFbit = true;
2133         boolean replayDone = false;
2134         final int numberOfStartBgpRetries = 3;
2135         RetryOnException startBgpRetry = new RetryOnException(numberOfStartBgpRetries);
2136         do {
2137             try {
2138                 LOG.debug("Replaying BGPConfig ");
2139                 br.startBgp(asNum, rid, stalepathTime, announceFbit);
2140                 LOG.debug("Replay BGPConfig successful");
2141                 replayDone = true;
2142                 break;
2143             } catch (BgpRouterException bre) {
2144                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
2145                     LOG.debug("Starting the routesync for exception", bre);
2146                     startBgpRetry.errorOccured();
2147                     if (!startBgpRetry.shouldRetry()) {
2148                         LOG.debug("starting route sync for BgpRouter exception");
2149                         doRouteSync();
2150                     }
2151                 } else {
2152                     LOG.error("Replay: startBgp() received exception error {} : ",
2153                             bre.getErrorCode(), bre);
2154                     startBgpRetry.errorOccured();
2155                 }
2156             } catch (TApplicationException tae) {
2157                 if (tae.getType() == BgpRouterException.BGP_ERR_ACTIVE) {
2158                     LOG.debug("Starting the routesync for exception", tae);
2159                     startBgpRetry.errorOccured();
2160                     if (!startBgpRetry.shouldRetry()) {
2161                         LOG.debug("starting route sync for Thrift BGP_ERR_ACTIVE exception");
2162                         doRouteSync();
2163                     }
2164                 } else if (tae.getType() == BgpRouterException.BGP_ERR_COMMON_FAILURE) {
2165                     LOG.debug("Starting the routesync for AS-ID started exception", tae);
2166                     startBgpRetry.errorOccured();
2167                     if (!startBgpRetry.shouldRetry()) {
2168                         LOG.debug("starting route sync for Thrift BGP_ERR_COMMON_FAILURE exception");
2169                         doRouteSync();
2170                     }
2171                 } else {
2172                     LOG.error("Replay: startBgp() received exception type {}: ",
2173                             tae.getType(), tae);
2174                     startBgpRetry.errorOccured();
2175                 }
2176             } catch (Exception e) {
2177                 //not unusual. We may have restarted & BGP is already on
2178                 LOG.error("Replay:startBgp() received exception: ", e);
2179                 startBgpRetry.errorOccured();
2180             }
2181         } while (startBgpRetry.shouldRetry());
2182
2183         replaySucceded = replayDone;
2184
2185         startBgpCountersTask();
2186         startBgpAlarmsTask();
2187
2188         /*
2189          * commenting this due to a bug with QBGP. Will uncomment once QBGP fix is done.
2190          * This wont have any functional impacts
2191          */
2192         try {
2193             br.delayEOR(delayEorSeconds);
2194         } catch (TException | BgpRouterException e) {
2195             LOG.error("Replay: delayEOR() number of seconds to wait for EOR from ODL:", e);
2196         }
2197
2198         List<Neighbors> neighbors = config.getNeighbors();
2199         if (neighbors != null) {
2200             LOG.error("configuring existing Neighbors present for replay total neighbors {}", neighbors.size());
2201             boolean neighborConfigReplayResult = replayNbrConfig(neighbors, br);
2202             if (neighborConfigReplayResult == false) {
2203                 replaySucceded = false;
2204             }
2205         } else {
2206             LOG.error("no Neighbors present for replay config ");
2207         }
2208
2209         Logging logging = config.getLogging();
2210         if (logging != null) {
2211             try {
2212                 br.setLogging(logging.getFile(), logging.getLevel());
2213             } catch (TException | BgpRouterException e) {
2214                 LOG.error("Replay:setLogging() received exception", e);
2215             }
2216         }
2217
2218         GracefulRestart gracefulRestart = config.getGracefulRestart();
2219         if (gracefulRestart != null) {
2220             try {
2221                 br.addGracefulRestart(gracefulRestart.getStalepathTime().intValue());
2222             } catch (TException | BgpRouterException e) {
2223                 LOG.error("Replay:addGr() received exception", e);
2224             }
2225         }
2226
2227         List<Vrfs> vrfs = config.getVrfs();
2228         if (vrfs == null) {
2229             vrfs = new ArrayList<>();
2230         }
2231         for (Vrfs vrf : vrfs) {
2232             for (AddressFamiliesVrf adf : vrf.getAddressFamiliesVrf()) {
2233                 try {
2234                     br.addVrf(BgpUtil.getLayerType(adf), vrf.getRd(), vrf.getImportRts(),
2235                             vrf.getExportRts());
2236                 } catch (TException | BgpRouterException e) {
2237                     LOG.error("Replay:addVrf() received exception", e);
2238                 }
2239             }
2240         }
2241
2242         List<Networks> ln = config.getNetworks();
2243         if (ln != null) {
2244             for (Networks net : ln) {
2245                 String rd = net.getRd();
2246                 String pfxlen = net.getPrefixLen();
2247                 String nh = net.getNexthop().getValue();
2248                 Long label = net.getLabel();
2249                 int lbl = label == null ? 0 : label.intValue();
2250                 int l3vni = net.getL3vni() == null ? 0 : net.getL3vni().intValue();
2251                 int l2vni = net.getL2vni() == null ? 0 : net.getL2vni().intValue();
2252                 if (rd == null && lbl > 0) {
2253                     //LU prefix is being deleted.
2254                     rd = Integer.toString(lbl);
2255                 }
2256
2257                 BgpControlPlaneType protocolType = net.getBgpControlPlaneType();
2258                 int ethernetTag = net.getEthtag().intValue();
2259                 String esi = net.getEsi();
2260                 String macaddress = net.getMacaddress();
2261                 EncapType encapType = net.getEncapType();
2262                 String routerMac = net.getRoutermac();
2263
2264                 try {
2265                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni,
2266                             BgpUtil.convertToThriftProtocolType(protocolType),
2267                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
2268                 } catch (TException | BgpRouterException e) {
2269                     LOG.error("Replay:addPfx() received exception", e);
2270                 }
2271             }
2272         }
2273
2274
2275         List<Multipath> multipaths = config.getMultipath();
2276
2277         if (multipaths != null) {
2278             for (Multipath multipath : multipaths) {
2279                 if (multipath != null) {
2280                     af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
2281                     af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
2282
2283                     try {
2284                         if (multipath.isMultipathEnabled()) {
2285                             br.enableMultipath(afi, safi);
2286                         } else {
2287                             br.disableMultipath(afi, safi);
2288                         }
2289                     } catch (TException | BgpRouterException e) {
2290                         LOG.info("Replay:multipaths() received exception", e);
2291                     }
2292                 }
2293             }
2294         }
2295         List<VrfMaxpath> vrfMaxpaths = config.getVrfMaxpath();
2296         if (vrfMaxpaths != null) {
2297             for (VrfMaxpath vrfMaxpath : vrfMaxpaths) {
2298                 try {
2299                     br.multipaths(vrfMaxpath.getRd(), vrfMaxpath.getMaxpaths());
2300                 } catch (TException | BgpRouterException e) {
2301                     LOG.info("Replay:vrfMaxPath() received exception", e);
2302                 }
2303             }
2304         }
2305
2306         //send End of Rib Marker to Qthriftd.
2307         final int numberOfEORRetries = 3;
2308         replayDone = false;
2309         RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
2310         do {
2311             try {
2312                 br.sendEOR();
2313                 LOG.debug("Replay sendEOR {} successful");
2314                 replayDone = true;
2315                 break;
2316             } catch (Exception e) {
2317                 eorRetry.errorOccured();
2318                 LOG.error("Replay:sedEOR() received exception:", e);
2319             }
2320         } while (eorRetry.shouldRetry());
2321
2322         return replaySucceded && replayDone;
2323     }
2324
2325     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
2326         bgpUtil.update(iid, dto);
2327     }
2328
2329     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
2330         bgpUtil.delete(iid);
2331     }
2332
2333     public void startConfig(String bgpHost, int thriftPort) {
2334         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2335                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2336         InstanceIdentifier<ConfigServer> iid = iib.build();
2337         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
2338         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
2339                 .setPort((long) thriftPort).build();
2340         update(iid, dto);
2341     }
2342
2343     public void startBgp(long as, String routerId, int spt, boolean fbit) {
2344         IpAddress rid = routerId == null ? null : IpAddressBuilder.getDefaultInstance(routerId);
2345         Long staleTime = (long) spt;
2346         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2347                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2348         InstanceIdentifier<AsId> iid = iib.build();
2349         AsId dto = new AsIdBuilder().setLocalAs(as)
2350                 .setRouterId(rid)
2351                 .setStalepathTime(staleTime)
2352                 .setAnnounceFbit(fbit).build();
2353         update(iid, dto);
2354     }
2355
2356     public void addLogging(String fileName, String logLevel) {
2357         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2358                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2359         InstanceIdentifier<Logging> iid = iib.build();
2360         Logging dto = new LoggingBuilder().setFile(fileName)
2361                 .setLevel(logLevel).build();
2362         update(iid, dto);
2363     }
2364
2365     public void addGracefulRestart(int staleTime) {
2366         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2367                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
2368         InstanceIdentifier<GracefulRestart> iid = iib.build();
2369         GracefulRestart dto = new GracefulRestartBuilder()
2370                 .setStalepathTime((long) staleTime).build();
2371         update(iid, dto);
2372     }
2373
2374     public void addNeighbor(
2375             String nbrIp, long remoteAs, @Nullable final TcpMd5SignaturePasswordType md5Secret) {
2376         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2377         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2378                 InstanceIdentifier.builder(Bgp.class)
2379                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2380         InstanceIdentifier<Neighbors> iid = iib.build();
2381         TcpSecurityOption tcpSecOption = null;
2382         if (md5Secret != null) {
2383             tcpSecOption = new TcpMd5SignatureOptionBuilder().setTcpMd5SignaturePassword(md5Secret).build();
2384         } // else let tcpSecOption be null
2385         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
2386                 .setRemoteAs(remoteAs).setTcpSecurityOption(tcpSecOption).build();
2387         update(iid, dto);
2388     } // public addNeighbor(nbrIp, remoteAs, md5Secret)
2389
2390     public void addUpdateSource(String nbrIp, String srcIp) {
2391         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2392         Ipv4Address srcAddr = new Ipv4Address(srcIp);
2393         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2394                 InstanceIdentifier.builder(Bgp.class)
2395                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2396                         .child(UpdateSource.class);
2397         InstanceIdentifier<UpdateSource> iid = iib.build();
2398         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
2399                 .setSourceIp(srcAddr).build();
2400         update(iid, dto);
2401     }
2402
2403     public void addEbgpMultihop(String nbrIp, int hops) {
2404         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2405         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2406                 InstanceIdentifier.builder(Bgp.class)
2407                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2408                         .child(EbgpMultihop.class);
2409         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2410         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
2411                 .setNhops((long) hops).build();
2412         update(iid, dto);
2413     }
2414
2415     public void addAddressFamily(String nbrIp, int afi, int safi) {
2416         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2417         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2418                 InstanceIdentifier.builder(Bgp.class)
2419                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2420                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2421         InstanceIdentifier<AddressFamilies> iid = iib.build();
2422         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
2423                 .setAfi((long) afi).setSafi((long) safi).build();
2424         update(iid, dto);
2425     }
2426
2427     public void addPrefix(String rd, String macAddress, String pfx, List<String> nhList,
2428               VrfEntry.EncapType encapType, long lbl, long l3vni, long l2vni, String gatewayMac) {
2429         for (String nh : nhList) {
2430             Ipv4Address nexthop = nh != null ? new Ipv4Address(nh) : null;
2431             Long label = lbl;
2432             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
2433                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
2434             NetworksBuilder networksBuilder = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
2435                                                 .setLabel(label).setEthtag(BgpConstants.DEFAULT_ETH_TAG);
2436             buildVpnEncapSpecificInfo(networksBuilder, encapType, label, l3vni, l2vni, macAddress, gatewayMac);
2437             update(iid, networksBuilder.build());
2438         }
2439     }
2440
2441     private static void buildVpnEncapSpecificInfo(NetworksBuilder builder, VrfEntry.EncapType encapType, long label,
2442                                                   long l3vni, long l2vni, String macAddress, String gatewayMac) {
2443         if (encapType.equals(VrfEntry.EncapType.Mplsgre)) {
2444             builder.setLabel(label).setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLL3VPN)
2445                     .setEncapType(EncapType.GRE);
2446         } else {
2447             builder.setL3vni(l3vni).setL2vni(l2vni).setMacaddress(macAddress).setRoutermac(gatewayMac)
2448                     .setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLEVPN).setEncapType(EncapType.VXLAN);
2449         }
2450     }
2451
2452     // TODO: add LayerType as arg - supports command
2453     public void addVrf(String rd, List<String> irts, List<String> erts, AddressFamily addressFamily) {
2454         Vrfs vrf = bgpUtil.getVrfFromRd(rd);
2455         List<AddressFamiliesVrf> adfList = new ArrayList<>(1);
2456         if (vrf != null) {
2457             adfList = vrf.getAddressFamiliesVrf();
2458         }
2459         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2460         if (addressFamily.equals(AddressFamily.IPV4)) {
2461             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2462             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2463         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2464             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2465             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2466         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2467             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2468             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2469         }
2470         AddressFamiliesVrf adf = adfBuilder.build();
2471         adfList.add(adf);
2472         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib = InstanceIdentifier.builder(Bgp.class)
2473                 .child(Vrfs.class, new VrfsKey(rd));
2474         InstanceIdentifier<Vrfs> iid = iib.build();
2475         Vrfs dto = new VrfsBuilder().setRd(rd).setImportRts(irts)
2476             .setExportRts(erts).setAddressFamiliesVrf(adfList).build();
2477
2478         List<AddressFamiliesVrf> listAdFamilies = mapNewAdFamily.get(rd);
2479         if (listAdFamilies != null) {
2480             listAdFamilies.add(adf);
2481         } else {
2482             mapNewAdFamily.put(rd, adfList);
2483         }
2484
2485         try {
2486             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
2487         } catch (TransactionCommitFailedException e) {
2488             LOG.error("Error adding VRF to datastore", e);
2489             throw new RuntimeException(e);
2490         }
2491     }
2492
2493     public void stopConfig() {
2494         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2495                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2496         InstanceIdentifier<ConfigServer> iid = iib.build();
2497         delete(iid);
2498     }
2499
2500     public void stopBgp() {
2501         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2502                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2503         InstanceIdentifier<AsId> iid = iib.build();
2504         delete(iid);
2505     }
2506
2507     public void delLogging() {
2508         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2509                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2510         InstanceIdentifier<Logging> iid = iib.build();
2511         delete(iid);
2512     }
2513
2514     public void delGracefulRestart() {
2515         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2516                 InstanceIdentifier.builder(Bgp.class)
2517                         .child(GracefulRestart.class);
2518         InstanceIdentifier<GracefulRestart> iid = iib.build();
2519         delete(iid);
2520     }
2521
2522     public void delNeighbor(String nbrIp) {
2523         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2524         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2525                 InstanceIdentifier.builder(Bgp.class)
2526                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2527         InstanceIdentifier<Neighbors> iid = iib.build();
2528         delete(iid);
2529     }
2530
2531     public void delUpdateSource(String nbrIp) {
2532         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2533         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2534                 InstanceIdentifier.builder(Bgp.class)
2535                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2536                         .child(UpdateSource.class);
2537         InstanceIdentifier<UpdateSource> iid = iib.build();
2538         delete(iid);
2539     }
2540
2541     public void delEbgpMultihop(String nbrIp) {
2542         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2543         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2544                 InstanceIdentifier.builder(Bgp.class)
2545                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2546                         .child(EbgpMultihop.class);
2547         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2548         delete(iid);
2549     }
2550
2551     public void delAddressFamily(String nbrIp, int afi, int safi) {
2552         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2553         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2554                 InstanceIdentifier.builder(Bgp.class)
2555                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2556                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2557         InstanceIdentifier<AddressFamilies> iid = iib.build();
2558         delete(iid);
2559     }
2560
2561     public void delPrefix(String rd, String pfx) {
2562         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
2563                 InstanceIdentifier.builder(Bgp.class)
2564                         .child(Networks.class, new NetworksKey(pfx, rd));
2565         InstanceIdentifier<Networks> iid = iib.build();
2566         delete(iid);
2567     }
2568
2569     public boolean delVrf(String rd, AddressFamily addressFamily) {
2570         if (addressFamily == null) {
2571             LOG.error("delVrf: vrf {}, addressFamily invalid", rd);
2572             return false;
2573         }
2574         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2575         if (addressFamily.equals(AddressFamily.IPV4)) {
2576             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2577             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2578         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2579             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2580             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2581         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2582             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2583             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2584         }
2585         Vrfs vrfOriginal = bgpUtil.getVrfFromRd(rd);
2586         if (vrfOriginal == null) {
2587             LOG.error("delVrf: no vrf with existing rd {}. step aborted", rd);
2588             return false;
2589         }
2590
2591         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2592                 InstanceIdentifier.builder(Bgp.class)
2593                         .child(Vrfs.class, new VrfsKey(rd));
2594
2595         InstanceIdentifier<Vrfs> iid = iib.build();
2596
2597         InstanceIdentifier.builder(Bgp.class).build()
2598                 .child(Multipath.class, new MultipathKey(adfBuilder.getAfi(), adfBuilder.getSafi()));
2599         @SuppressWarnings("static-access")
2600         InstanceIdentifier<Bgp> iid6 =  InstanceIdentifier.create(Bgp.class);
2601         InstanceIdentifierBuilder<Vrfs> iib3 = iid6.child(Vrfs.class, new VrfsKey(rd)).builder();
2602         InstanceIdentifier<Vrfs> iidFinal = iib3.build();
2603
2604         //** update or delete the vrfs with the rest of AddressFamilies already present in the last list
2605         AddressFamiliesVrf adfToDel = adfBuilder.build();
2606         List<AddressFamiliesVrf> adfListOriginal = vrfOriginal.getAddressFamiliesVrf() == null
2607                 ? new ArrayList<>() : vrfOriginal.getAddressFamiliesVrf();
2608         List<AddressFamiliesVrf> adfListToRemoveFromOriginal = new ArrayList<>();
2609         adfListOriginal.forEach(adf -> {
2610             if (adf.equals(adfToDel)) {
2611                 adfListToRemoveFromOriginal.add(adfToDel);
2612                 return;
2613             }
2614         });
2615         for (AddressFamiliesVrf adfToRemove : adfListToRemoveFromOriginal) {
2616             adfListOriginal.remove(adfToRemove);
2617             try {
2618                 SingleTransactionDataBroker.syncWrite(dataBroker,
2619                         LogicalDatastoreType.CONFIGURATION, iid, vrfOriginal);
2620             } catch (TransactionCommitFailedException e) {
2621                 LOG.error("delVrf: Error updating VRF to datastore", e);
2622                 throw new RuntimeException(e);
2623             }
2624         }
2625         if (adfListOriginal.isEmpty()) {
2626             delete(iidFinal);
2627             return true;
2628         }
2629         // not all is removed
2630         return false;
2631     }
2632
2633     public void setMultipathStatus(af_afi afi, af_safi safi, boolean enable) {
2634         long lafi = afi.getValue();
2635         long lsafi = safi.getValue();
2636
2637         InstanceIdentifier.InstanceIdentifierBuilder<Multipath> iib =
2638                 InstanceIdentifier
2639                         .builder(Bgp.class)
2640                         .child(Multipath.class,
2641                                 new MultipathKey(Long.valueOf(afi.getValue()), Long.valueOf(safi.getValue())));
2642
2643         Multipath dto = new MultipathBuilder().setAfi(lafi).setSafi(lsafi).setMultipathEnabled(enable).build();
2644         update(iib.build(), dto);
2645     }
2646
2647     public void multipaths(String rd, int maxpath) {
2648         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
2649                 InstanceIdentifier
2650                         .builder(Bgp.class)
2651                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
2652
2653         VrfMaxpath dto = new VrfMaxpathBuilder().setRd(rd).setMaxpaths(maxpath).build();
2654         update(iib.build(), dto);
2655     }
2656
2657     /*
2658     * Remove Stale Marked Routes after timer expiry.
2659     */
2660     private class RouteCleanup implements Callable<Integer> {
2661
2662         @Override
2663         public Integer call() {
2664             totalCleared = 0;
2665             try {
2666                 if (staledFibEntriesMap.isEmpty()) {
2667                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
2668                 } else {
2669                     for (String rd : staledFibEntriesMap.keySet()) {
2670                         if (Thread.interrupted()) {
2671                             return 0;
2672                         }
2673                         Map<String, Long> map = staledFibEntriesMap.get(rd);
2674                         if (map != null) {
2675                             for (String key : map.keySet()) {
2676                                 if (Thread.interrupted()) {
2677                                     return 0;
2678                                 }
2679                                 String prefix = extractPrefix(key);
2680                                 String nextHop = extractNextHop(key);
2681                                 totalCleared++;
2682                                 LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}, nextHop:{}",
2683                                         rd, prefix, nextHop);
2684                                 fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix, nextHop);
2685                             }
2686                         }
2687                     }
2688                 }
2689             } finally {
2690                 staledFibEntriesMap.clear();
2691             }
2692             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
2693             return 0;
2694         }
2695     }
2696
2697     /*
2698      * BGP restart scenario, ODL-BGP manager was/is running.
2699      * On re-sync notification, Get a copy of FIB database.
2700      */
2701     public void createStaleFibMap() {
2702         totalStaledCount = 0;
2703         try {
2704             staledFibEntriesMap.clear();
2705             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2706
2707             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2708                     LogicalDatastoreType.CONFIGURATION, id);
2709             if (fibEntries.isPresent()) {
2710                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2711                 for (VrfTables vrfTable : staleVrfTables) {
2712                     Map<String, Long> staleFibEntMap = new HashMap<>();
2713                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2714                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2715                             //Stale marking and cleanup is only meant for the routes learned through BGP.
2716                             continue;
2717                         }
2718                         if (Thread.interrupted()) {
2719                             break;
2720                         }
2721                         totalStaledCount++;
2722                         //Create MAP from staleVrfTables.
2723                         vrfEntry.getRoutePaths()
2724                                 .forEach(
2725                                     routePath -> staleFibEntMap.put(
2726                                             appendNextHopToPrefix(vrfEntry.getDestPrefix(),
2727                                                     routePath.getNexthopAddress()), routePath.getLabel()));
2728                     }
2729                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), staleFibEntMap);
2730                 }
2731             } else {
2732                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
2733             }
2734         } catch (ReadFailedException e) {
2735             LOG.error("createStaleFibMap:: error ", e);
2736         }
2737         LOG.error("created {} staled entries ", totalStaledCount);
2738     }
2739
2740     /*
2741      * BGP config remove scenario, Need to remove all the
2742      * external routes from FIB.
2743      */
2744     public void deleteExternalFibRoutes() {
2745         totalExternalRoutes = 0;
2746         totalExternalMacRoutes = 0;
2747         try {
2748             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2749
2750             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2751                     LogicalDatastoreType.CONFIGURATION, id);
2752             if (fibEntries.isPresent()) {
2753                 if (fibEntries.get().getVrfTables() == null) {
2754                     LOG.error("deleteExternalFibRoutes::getVrfTables is null");
2755                     return;
2756                 }
2757                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2758                 for (VrfTables vrfTable : staleVrfTables) {
2759                     String rd = vrfTable.getRouteDistinguisher();
2760                     if (vrfTable.getVrfEntry() != null) {
2761                         for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2762                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2763                                 //route cleanup is only meant for the routes learned through BGP.
2764                                 continue;
2765                             }
2766                             totalExternalRoutes++;
2767                             fibDSWriter.removeFibEntryFromDS(rd, vrfEntry.getDestPrefix());
2768                         }
2769                     } else if (vrfTable.getMacVrfEntry() != null) {
2770                         for (MacVrfEntry macEntry : vrfTable.getMacVrfEntry()) {
2771                             if (RouteOrigin.value(macEntry.getOrigin()) != RouteOrigin.BGP) {
2772                                 //route cleanup is only meant for the routes learned through BGP.
2773                                 continue;
2774                             }
2775                             totalExternalMacRoutes++;
2776                             fibDSWriter.removeMacEntryFromDS(rd, macEntry.getMac());
2777                         }
2778                     }
2779                 }
2780             } else {
2781                 LOG.error("deleteExternalFibRoutes:: FIBentries.class is not present");
2782             }
2783         } catch (ReadFailedException e) {
2784             LOG.error("deleteExternalFibRoutes:: error ", e);
2785         }
2786         LOG.debug("deleted {} fib entries {} mac entries", totalExternalRoutes, totalExternalMacRoutes);
2787     }
2788
2789     public boolean addToRt2TepMap(String rd, String tepIp, String mac, Long l2vni) {
2790         boolean isFirstMacUpdateFromTep = false;
2791         if (rt2TepMap.containsKey(rd)) {
2792             if (rt2TepMap.get(rd).containsKey(tepIp)) {
2793                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and tep-ip {}. No Elan DS write required",
2794                         mac, l2vni, rd, tepIp);
2795                 rt2TepMap.get(rd).get(tepIp).put(mac, l2vni);
2796             } else {
2797                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and new tep-ip {}",
2798                         mac, l2vni, rd, tepIp);
2799                 isFirstMacUpdateFromTep = true;
2800                 Map<String, Long> macList = new HashMap<>();
2801                 macList.put(mac, l2vni);
2802                 rt2TepMap.get(rd).put(tepIp, macList);
2803             }
2804         } else {
2805             LOG.debug("RT2 with mac {} l2vni {} from new rd {} and tep ip {}",
2806                     mac, l2vni, rd, tepIp);
2807             isFirstMacUpdateFromTep = true;
2808             Map<String, Long> macList = new HashMap<>();
2809             macList.put(mac, l2vni);
2810             Map<String, Map<String, Long>> tepIpMacMap = new HashMap<>();
2811             tepIpMacMap.put(tepIp, macList);
2812             rt2TepMap.put(rd, tepIpMacMap);
2813         }
2814         return isFirstMacUpdateFromTep;
2815     }
2816
2817     public boolean deleteFromRt2TepMap(String rd, String tepIp, String mac) {
2818         boolean isLastMacUpdateFromTep = false;
2819         LOG.debug("RT2 withdraw with rd {} mac {} tep-ip {} ", rd, mac, tepIp);
2820         if (rt2TepMap.containsKey(rd)) {
2821             if (rt2TepMap.get(rd).containsKey(tepIp)) {
2822                 if (rt2TepMap.get(rd).get(tepIp).containsKey(mac)) {
2823                     LOG.debug("RT2 Withdraw : Removing the mac {} from Map", mac);
2824                     rt2TepMap.get(rd).get(tepIp).remove(mac);
2825                     if (rt2TepMap.get(rd).get(tepIp).isEmpty()) {
2826                         isLastMacUpdateFromTep = true;
2827                         LOG.debug("RT2 Withdraw : Removing the tep-ip {} from Map", tepIp);
2828                         rt2TepMap.get(rd).remove(tepIp);
2829                         if (rt2TepMap.get(rd).isEmpty()) {
2830                             LOG.debug("RT2 Withdraw : Removing the rd {} from Map", rd);
2831                             rt2TepMap.remove(rd);
2832                         }
2833                     }
2834                 }
2835             }
2836         }
2837         return isLastMacUpdateFromTep;
2838     }
2839
2840     public Collection<String> getTepIPs(String rd) {
2841         final Map<String, Map<String, Long>> tepIpMap = rt2TepMap.get(rd);
2842         return tepIpMap != null ? tepIpMap.keySet() : Collections.emptyList();
2843     }
2844
2845     public boolean isBgpConnected() {
2846         return bgpRouter.isBgpConnected();
2847     }
2848
2849     public long getLastConnectedTS() {
2850         return bgpRouter.getLastConnectedTS();
2851     }
2852
2853     public long getConnectTS() {
2854         return bgpRouter.getConnectTS();
2855     }
2856
2857     public long getStartTS() {
2858         return bgpRouter.getStartTS();
2859     }
2860
2861     public TTransport getTransport() {
2862         return bgpRouter.getTransport();
2863     }
2864
2865     public int getTotalStaledCount() {
2866         return totalStaledCount;
2867     }
2868
2869     public int getTotalCleared() {
2870         return totalCleared;
2871     }
2872
2873     public BgpCounters getBgpCounters() {
2874         return bgpCountersReference.get();
2875     }
2876
2877     private void startBgpCountersTask() {
2878         if (getBgpCounters() == null && bgpCountersReference.compareAndSet(null,
2879                 new BgpCounters(getBgpSdncMipIp(), metricProvider))) {
2880             bgpCountersTask = executor.scheduleAtFixedRate(bgpCountersReference.get(), 0, 120 * 1000,
2881                     TimeUnit.MILLISECONDS);
2882             LOG.info("Bgp Counters task scheduled for every two minutes.");
2883
2884             bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
2885         }
2886     }
2887
2888     private void stopBgpCountersTask() {
2889         final BgpCounters bgpCounters = bgpCountersReference.getAndSet(null);
2890         if (bgpCounters != null) {
2891             bgpCountersTask.cancel(true);
2892             bgpCounters.close();
2893         }
2894     }
2895
2896     private void startBgpAlarmsTask() {
2897         if (getBgpAlarms() == null && bgpAlarmsReference.compareAndSet(null, new BgpAlarms(this))) {
2898             bgpAlarmsReference.get().init();
2899             bgpAlarmsTask = executor.scheduleAtFixedRate(bgpAlarmsReference.get(), 0, 60 * 1000, TimeUnit.MILLISECONDS);
2900             LOG.info("Bgp Alarms task scheduled for every minute.");
2901         }
2902     }
2903
2904     private void stopBgpAlarmsTask() {
2905         final BgpAlarms bgpAlarms = bgpAlarmsReference.getAndSet(null);
2906         if (bgpAlarms != null) {
2907             bgpAlarmsTask.cancel(true);
2908             bgpAlarms.close();
2909         }
2910     }
2911
2912     public BgpAlarms getBgpAlarms() {
2913         return bgpAlarmsReference.get();
2914     }
2915
2916     private static String appendNextHopToPrefix(String prefix, String nextHop) {
2917         return prefix + ":" + nextHop;
2918     }
2919
2920     private static String extractPrefix(String prefixNextHop) {
2921         return prefixNextHop.split(":")[0];
2922     }
2923
2924     private static String extractNextHop(String prefixNextHop) {
2925         return prefixNextHop.split(":")[1];
2926     }
2927
2928     private static String extractMd5Secret(final Neighbors val) {
2929         String md5Secret = null;
2930         TcpSecurityOption tcpSecOpt = val.getTcpSecurityOption();
2931         if (tcpSecOpt != null) {
2932             if (tcpSecOpt instanceof TcpMd5SignatureOption) {
2933                 md5Secret = ((TcpMd5SignatureOption) tcpSecOpt).getTcpMd5SignaturePassword().getValue();
2934             } else { // unknown TcpSecurityOption
2935                 LOG.debug("neighbors  Ignored unknown tcp-security-option of peer {}", val.getAddress().getValue());
2936             }
2937         }
2938         return md5Secret;
2939     } // private method extractMd5Secret
2940 }