bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.net.InetAddresses;
  13 import com.google.common.util.concurrent.ThreadFactoryBuilder;
  14 import io.netty.util.concurrent.GlobalEventExecutor;
  15 import java.io.IOException;
  16 import java.lang.reflect.Constructor;
  17 import java.lang.reflect.InvocationTargetException;
  18 import java.net.InetAddress;
  19 import java.net.InetSocketAddress;
  20 import java.net.NetworkInterface;
  21 import java.net.SocketException;
  22 import java.util.ArrayList;
  23 import java.util.Collection;
  24 import java.util.Collections;
  25 import java.util.Enumeration;
  26 import java.util.HashMap;
  27 import java.util.Iterator;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Timer;
  32 import java.util.concurrent.Callable;
  33 import java.util.concurrent.ConcurrentHashMap;
  34 import java.util.concurrent.CountDownLatch;
  35 import java.util.concurrent.ExecutionException;
  36 import java.util.concurrent.Executors;
  37 import java.util.concurrent.Future;
  38 import java.util.concurrent.ScheduledExecutorService;
  39 import java.util.concurrent.ScheduledFuture;
  40 import java.util.concurrent.TimeUnit;
  41 import java.util.concurrent.TimeoutException;
  42 import java.util.concurrent.atomic.AtomicInteger;
  43 import java.util.concurrent.atomic.AtomicReference;
  44 import javax.annotation.Nullable;
  45 import javax.annotation.PreDestroy;
  46 import javax.inject.Inject;
  47 import javax.inject.Singleton;
  48 import org.apache.thrift.TApplicationException;
  49 import org.apache.thrift.TException;
  50 import org.apache.thrift.transport.TTransport;
  51 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  52 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  53 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  54 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  55 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  56 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  57 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  60 import org.opendaylight.infrautils.metrics.MetricProvider;
  61 import org.opendaylight.mdsal.eos.binding.api.Entity;
  62 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipCandidateRegistration;
  63 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipListenerRegistration;
  64 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  65 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  66 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  67 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  68 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  69 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  70 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  71 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  72 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  73 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  74 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  75 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  76 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  77 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  78 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  79 import org.opendaylight.netvirt.bgpmanager.thrift.gen.protocol_type;
  80 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  81 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  82 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  83 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  84 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.AddressFamily;
  85 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  86 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
  87 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EncapType;
  88 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.TcpMd5SignaturePasswordType;
  89 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
  90 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
  92 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
  94 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
  96 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Multipath;
  98 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathKey;
 100 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Neighbors;
 101 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsKey;
 103 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Networks;
 104 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksKey;
 106 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpath;
 107 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathKey;
 109 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Vrfs;
 110 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsKey;
 112 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamilies;
 113 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesKey;
 115 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihop;
 116 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihopBuilder;
 117 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSource;
 118 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSourceBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfs.AddressFamiliesVrf;
 120 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfs.AddressFamiliesVrfBuilder;
 121 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.TcpSecurityOption;
 122 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOption;
 123 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOptionBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
 125 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.macvrfentries.MacVrfEntry;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 131 import org.opendaylight.yangtools.yang.binding.DataObject;
 132 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 133 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 134 import org.osgi.framework.BundleContext;
 135 import org.osgi.util.tracker.ServiceTracker;
 136 import org.slf4j.Logger;
 137 import org.slf4j.LoggerFactory;
 138
 139 @Singleton
 140 public class BgpConfigurationManager {
 141     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 142
 143     // to have stale FIB map (RD, Prefix)
 144     //  number of seconds wait for route sync-up between ODL and BGP
 145     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 146     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 147     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 148     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 149     private static final String DEF_UPORT = "6644";
 150     private static final String DEF_CHOST = "255.255.255.255"; // Invalid Host IP
 151     private static final String DEF_CPORT = "7644";
 152     private static final String DEF_BGP_SDNC_MIP = "127.0.0.1";
 153     private static final String BGP_SDNC_MIP = "vpnservice.bgp.thrift.sdnc.mip";
 154     private static final int RESTART_DEFAULT_GR = 90;
 155     private static final int DS_RETRY_COUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
 156     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
 157     private static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 158     private static final String BGP_EOR_DELAY = "vpnservice.bgp.eordelay";
 159     private static final String DEF_BGP_EOR_DELAY = "1800";
 160     private static final String BGP_ENTITY_NAME = "bgp";
 161     private static final String ADD_WARN = "Config store updated; undo with Delete if needed.";
 162     private static final String DEL_WARN = "Config store updated; undo with Add if needed.";
 163     private static final String UPD_WARN = "Update operation not supported; Config store updated;"
 164             + " restore with another Update if needed.";
 165
 166     private static final Class<?>[] REACTORS = {
 167         ConfigServerReactor.class, AsIdReactor.class,
 168         GracefulRestartReactor.class, LoggingReactor.class,
 169         NeighborsReactor.class, UpdateSourceReactor.class,
 170         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 171         NetworksReactor.class, VrfsReactor.class, BgpReactor.class,
 172         MultipathReactor.class, VrfMaxpathReactor.class
 173     };
 174
 175     private IBgpManager bgpManager;
 176     private final DataBroker dataBroker;
 177     private final FibDSWriter fibDSWriter;
 178     private final IVpnLinkService vpnLinkService;
 179     private final BundleContext bundleContext;
 180     private final BgpUtil bgpUtil;
 181     private volatile Bgp config;
 182     private final BgpRouter bgpRouter;
 183     private final BgpSyncHandle bgpSyncHandle = new BgpSyncHandle();
 184     private volatile BgpThriftService bgpThriftService = null;
 185     private final int delayEorSeconds;
 186
 187     private final CountDownLatch initer = new CountDownLatch(1);
 188
 189     private final String hostStartup;
 190     private final String portStartup;
 191
 192     private final AtomicReference<BgpCounters> bgpCountersReference = new AtomicReference<>();
 193     private ScheduledFuture<?> bgpCountersTask;
 194
 195     private final AtomicReference<BgpAlarms> bgpAlarmsReference = new AtomicReference<>();
 196     private ScheduledFuture<?> bgpAlarmsTask;
 197
 198     private Future<?> lastReplayJobFt;
 199     private ScheduledFuture<?> routeCleanupFuture;
 200
 201     private long staleStartTime;
 202     private long staleEndTime;
 203     private long cfgReplayStartTime;
 204     private long cfgReplayEndTime;
 205     private long staleCleanupTime;
 206     private int totalStaledCount;
 207     private int totalCleared;
 208     private int totalExternalRoutes;
 209     private int totalExternalMacRoutes;
 210
 211     private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(
 212             new ThreadFactoryBuilder().setNameFormat("BgpConfigurationManager-%d").setDaemon(true).build());
 213
 214     /**
 215      * this map store the new address families to send to quagga. When it is sended you must clear it.
 216      * The keys String are rd (route distinguisher).
 217      * */
 218     private final ConcurrentHashMap<String, List<AddressFamiliesVrf>> mapNewAdFamily = new ConcurrentHashMap<>();
 219
 220     // map<rd, map<prefix/len:nexthop, label>>
 221     private final Map<String, Map<String, Long>> staledFibEntriesMap = new ConcurrentHashMap<>();
 222
 223     // map<rd, map<tep-ip, map<mac, l2vni>>>
 224     private final Map<String, Map<String, Map<String, Long>>> rt2TepMap = new ConcurrentHashMap<>();
 225
 226     private final List<AutoCloseable> listeners = new ArrayList<>();
 227
 228     private final EntityOwnershipUtils entityOwnershipUtils;
 229     private final EntityOwnershipCandidateRegistration candidateRegistration;
 230     private final EntityOwnershipListenerRegistration entityListenerRegistration;
 231     private final MetricProvider metricProvider;
 232
 233     @Inject
 234     public BgpConfigurationManager(final DataBroker dataBroker,
 235             final EntityOwnershipService entityOwnershipService,
 236             final FibDSWriter fibDSWriter,
 237             final IVpnLinkService vpnLinkSrvce,
 238             final BundleContext bundleContext,
 239             final BgpUtil bgpUtil,
 240             final MetricProvider metricProvider) {
 241         this.dataBroker = dataBroker;
 242         this.fibDSWriter = fibDSWriter;
 243         this.vpnLinkService = vpnLinkSrvce;
 244         this.bundleContext = bundleContext;
 245         this.bgpUtil = bgpUtil;
 246         this.metricProvider = metricProvider;
 247         String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
 248         hostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 249         portStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 250         LOG.info("ConfigServer at {}:{}", hostStartup, portStartup);
 251         VtyshCli.setHostAddr(hostStartup);
 252         ClearBgpCli.setHostAddr(hostStartup);
 253         bgpRouter = BgpRouter.newInstance(this::getConfig, this::isBGPEntityOwner);
 254         delayEorSeconds = Integer.parseInt(getProperty(BGP_EOR_DELAY, DEF_BGP_EOR_DELAY));
 255         registerCallbacks();
 256
 257         entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 258
 259         candidateRegistration = registerEntityCandidate(entityOwnershipService);
 260         entityListenerRegistration = registerEntityListener(entityOwnershipService);
 261
 262         LOG.info("BGP Configuration manager initialized");
 263         initer.countDown();
 264
 265         GlobalEventExecutor.INSTANCE.execute(() -> {
 266             ServiceTracker<IBgpManager, ?> tracker = null;
 267             try {
 268                 tracker = new ServiceTracker<>(bundleContext, IBgpManager.class, null);
 269                 tracker.open();
 270                 bgpManager = (IBgpManager) tracker.waitForService(TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES));
 271                 Preconditions.checkState(bgpManager != null, "IBgpManager service not found");
 272             } catch (InterruptedException e) {
 273                 throw new IllegalStateException("Error retrieving IBgpManager service", e);
 274             } finally {
 275                 if (tracker != null) {
 276                     tracker.close();
 277                 }
 278             }
 279
 280             if (InetAddresses.isInetAddress(getBgpSdncMipIp())) {
 281                 InetSocketAddress bgpThriftServerSocketAddr = new InetSocketAddress(getBgpSdncMipIp(),
 282                         Integer.parseInt(updatePort));
 283                 bgpThriftService = new BgpThriftService(bgpThriftServerSocketAddr, bgpManager, this);
 284                 if (isBGPEntityOwner()) {
 285                     //I am EoS owner of BGP, opening bgp thrift UPDATE-SERVER port.
 286                     LOG.info("BGP Configuration manager initialized: UPDATE-SERVER started");
 287                     bgpThriftService.start();
 288                 }
 289                 LOG.info("UPDATE server started :ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 290             } else {
 291                 LOG.error("Failed to init UPDATE server invalid ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 292             }
 293             LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 294         });
 295     }
 296
 297     public String getBgpSdncMipIp() {
 298         return getProperty(BGP_SDNC_MIP, DEF_BGP_SDNC_MIP);
 299     }
 300
 301     public long getStaleCleanupTime() {
 302         return staleCleanupTime;
 303     }
 304
 305     public void setStaleCleanupTime(long staleCleanupTime) {
 306         this.staleCleanupTime = staleCleanupTime;
 307     }
 308
 309     public long getCfgReplayEndTime() {
 310         return cfgReplayEndTime;
 311     }
 312
 313     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 314         this.cfgReplayEndTime = cfgReplayEndTime;
 315     }
 316
 317     public long getCfgReplayStartTime() {
 318         return cfgReplayStartTime;
 319     }
 320
 321     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 322         this.cfgReplayStartTime = cfgReplayStartTime;
 323     }
 324
 325     public long getStaleEndTime() {
 326         return staleEndTime;
 327     }
 328
 329     public void setStaleEndTime(long staleEndTime) {
 330         this.staleEndTime = staleEndTime;
 331     }
 332
 333     public long getStaleStartTime() {
 334         return staleStartTime;
 335     }
 336
 337     public void setStaleStartTime(long staleStartTime) {
 338         this.staleStartTime = staleStartTime;
 339     }
 340
 341     private Object createListener(Class<?> cls) {
 342         try {
 343             Constructor<?> ctor = cls.getConstructor(BgpConfigurationManager.class);
 344             return ctor.newInstance(this);
 345         } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException
 346                 e) {
 347             LOG.error("Failed to create listener object", e);
 348             return null;
 349         }
 350     }
 351
 352     private void registerCallbacks() {
 353         for (Class<?> reactor : REACTORS) {
 354             Object obj = createListener(reactor);
 355             if (obj != null) {
 356                 AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 357                 dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 358                 listeners.add(dcl);
 359             }
 360         }
 361     }
 362
 363     @SuppressWarnings("checkstyle:IllegalCatch")
 364     @PreDestroy
 365     public void close() {
 366         executor.shutdown();
 367
 368         if (bgpThriftService != null) {
 369             bgpThriftService.stop();
 370             bgpThriftService = null;
 371         }
 372
 373         if (isBgpConnected()) {
 374             //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 375             bgpRouter.disconnect();
 376         }
 377
 378         if (candidateRegistration != null) {
 379             candidateRegistration.close();
 380         }
 381
 382         entityListenerRegistration.close();
 383
 384         listeners.forEach(l -> {
 385             try {
 386                 l.close();
 387             } catch (Exception e) {
 388                 LOG.warn("Error closing {}", l ,e);
 389             }
 390         });
 391
 392         LOG.info("{} close", getClass().getSimpleName());
 393     }
 394
 395     private String getProperty(String var, String def) {
 396         String property = bundleContext.getProperty(var);
 397         return property == null ? def : property;
 398     }
 399
 400     private EntityOwnershipCandidateRegistration registerEntityCandidate(
 401             final EntityOwnershipService entityOwnershipService) {
 402         try {
 403             return entityOwnershipService.registerCandidate(
 404                     new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME));
 405         } catch (CandidateAlreadyRegisteredException e) {
 406             LOG.error("failed to register bgp entity", e);
 407             return null;
 408         }
 409     }
 410
 411     private EntityOwnershipListenerRegistration registerEntityListener(
 412             final EntityOwnershipService entityOwnershipService) {
 413         return entityOwnershipService.registerListener(BGP_ENTITY_TYPE_FOR_OWNERSHIP, ownershipChange -> {
 414             LOG.trace("entity owner change event fired: {}", ownershipChange);
 415
 416             if (ownershipChange.getState() == EntityOwnershipChangeState.LOCAL_OWNERSHIP_GRANTED) {
 417                 LOG.trace("This PL is the Owner");
 418                 if (bgpThriftService != null) {
 419                     //opening UPDATE-SERVER port.
 420                     bgpThriftService.start();
 421                 } else {
 422                     LOG.error("I am the owner of BGP entity, but bgpThriftService is not initialized yet");
 423                 }
 424                 bgpRestarted();
 425             } else {
 426                 LOG.debug("Not owner: hasOwner: {}, isOwner: {}", ownershipChange.getState().hasOwner(),
 427                         ownershipChange.getState().isOwner());
 428                 if (bgpThriftService != null && bgpThriftService.isBgpThriftServiceStarted()) {
 429                     //close the bgp Thrift Update-SERVER port opened on non-Entity Owner
 430                     bgpThriftService.stop();
 431                     bgpThriftService = null;
 432                 }
 433                 if (isBgpConnected()) {
 434                     //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 435                     bgpRouter.disconnect();
 436                 }
 437             }
 438         });
 439     }
 440
 441     public boolean isBGPEntityOwner() {
 442         return entityOwnershipUtils.isEntityOwner(new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME), 0, 1);
 443     }
 444
 445     public Bgp get() {
 446         config = getConfig();
 447         return config;
 448     }
 449
 450     public class ConfigServerReactor
 451             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 452             implements ClusteredDataTreeChangeListener<ConfigServer> {
 453         private static final String YANG_OBJ = "config-server ";
 454
 455         public ConfigServerReactor() {
 456             super(ConfigServer.class, ConfigServerReactor.class);
 457         }
 458
 459         @Override
 460         protected void add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 461             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 462             if (!isBGPEntityOwner()) {
 463                 return;
 464             }
 465
 466             try {
 467                 initer.await();
 468             } catch (InterruptedException e) {
 469                 // Ignored
 470             }
 471             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 472             bgpRouter.configServerUpdated();
 473             synchronized (BgpConfigurationManager.this) {
 474                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 475                         val.getPort().intValue());
 476                 if (!res) {
 477                     LOG.error(YANG_OBJ + "Add failed; " + ADD_WARN);
 478                 }
 479             }
 480             VtyshCli.setHostAddr(val.getHost().getValue());
 481             ClearBgpCli.setHostAddr(val.getHost().getValue());
 482         }
 483
 484         @Override
 485         protected ConfigServerReactor getDataTreeChangeListener() {
 486             return ConfigServerReactor.this;
 487         }
 488
 489         @Override
 490         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 491             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 492         }
 493
 494         @Override
 495         protected void remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 496             LOG.trace("received bgp disconnect");
 497             if (!isBGPEntityOwner()) {
 498                 return;
 499             }
 500
 501             bgpRouter.configServerUpdated();
 502
 503             synchronized (BgpConfigurationManager.this) {
 504                 bgpRouter.disconnect();
 505             }
 506         }
 507
 508         @Override
 509         protected void update(InstanceIdentifier<ConfigServer> iid,
 510                 ConfigServer oldval, ConfigServer newval) {
 511             LOG.trace("received bgp Connection update");
 512             if (!isBGPEntityOwner()) {
 513                 return;
 514             }
 515             LOG.error(YANG_OBJ + UPD_WARN);
 516         }
 517     }
 518
 519     private BgpRouter getClient(String yangObj) {
 520         if (bgpRouter == null || !bgpRouter.isBgpConnected()) {
 521             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 522             return null;
 523         }
 524         return bgpRouter;
 525     }
 526
 527     public class AsIdReactor
 528             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 529             implements ClusteredDataTreeChangeListener<AsId> {
 530
 531         private static final String YANG_OBJ = "as-id ";
 532
 533         public AsIdReactor() {
 534             super(AsId.class, AsIdReactor.class);
 535         }
 536
 537         @Override
 538         protected void add(InstanceIdentifier<AsId> iid, AsId val) {
 539             LOG.error("received bgp add asid {}", val);
 540             if (!isBGPEntityOwner()) {
 541                 return;
 542             }
 543             LOG.debug("received add router config asNum {}", val.getLocalAs());
 544             synchronized (BgpConfigurationManager.this) {
 545                 BgpRouter br = getClient(YANG_OBJ);
 546                 if (br == null) {
 547                     LOG.debug("{} Unable to process add for asNum {}; {} {}", YANG_OBJ, val.getLocalAs(),
 548                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 549                     return;
 550                 }
 551                 bgpRestarted();
 552
 553                 startBgpCountersTask();
 554                 startBgpAlarmsTask();
 555             }
 556         }
 557
 558         @Override
 559         protected AsIdReactor getDataTreeChangeListener() {
 560             return AsIdReactor.this;
 561         }
 562
 563         @Override
 564         protected InstanceIdentifier<AsId> getWildCardPath() {
 565             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 566         }
 567
 568         @Override
 569         protected void remove(InstanceIdentifier<AsId> iid, AsId val) {
 570             LOG.error("received delete router config asNum {}", val.getLocalAs());
 571             if (!isBGPEntityOwner()) {
 572                 return;
 573             }
 574             synchronized (BgpConfigurationManager.this) {
 575                 long asNum = val.getLocalAs();
 576                 BgpRouter br = getClient(YANG_OBJ);
 577                 if (br == null) {
 578                     LOG.debug("{} Unable to process remove for asNum {}; {} {}", YANG_OBJ, asNum,
 579                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 580                     return;
 581                 }
 582                 try {
 583                     br.stopBgp(asNum);
 584                 } catch (TException | BgpRouterException e) {
 585                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 586                 }
 587
 588                 stopBgpCountersTask();
 589                 stopBgpAlarmsTask();
 590
 591                 Bgp conf = getConfig();
 592                 if (conf == null) {
 593                     LOG.error("Config Null while removing the as-id");
 594                     return;
 595                 }
 596                 LOG.debug("Removing external routes from FIB");
 597                 deleteExternalFibRoutes();
 598                 List<Neighbors> nbrs = conf.getNeighbors();
 599                 if (nbrs != null && nbrs.size() > 0) {
 600                     LOG.error("Tring to remove the as-id when neighbor config is already present");
 601                     for (Neighbors nbr : nbrs) {
 602                         LOG.debug("Removing Neighbor {} from Data store", nbr.getAddress().getValue());
 603                         delNeighbor(nbr.getAddress().getValue());
 604                     }
 605                 }
 606             }
 607         }
 608
 609         @Override
 610         protected void update(InstanceIdentifier<AsId> iid,
 611                 AsId oldval, AsId newval) {
 612             if (!isBGPEntityOwner()) {
 613                 return;
 614             }
 615             LOG.error(YANG_OBJ + UPD_WARN);
 616         }
 617     }
 618
 619     public class GracefulRestartReactor
 620             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 621             implements ClusteredDataTreeChangeListener<GracefulRestart> {
 622
 623         private static final String YANG_OBJ = "graceful-restart ";
 624
 625         public GracefulRestartReactor() {
 626             super(GracefulRestart.class, GracefulRestartReactor.class);
 627         }
 628
 629         @Override
 630         protected void add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 631             if (!isBGPEntityOwner()) {
 632                 return;
 633             }
 634             synchronized (BgpConfigurationManager.this) {
 635                 int stalePathTime = val.getStalepathTime().intValue();
 636                 BgpRouter br = getClient(YANG_OBJ);
 637                 if (br == null) {
 638                     LOG.error("{} Unable to add stale-path time {}; {} {}", YANG_OBJ, stalePathTime,
 639                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 640                     return;
 641                 }
 642                 try {
 643                     br.addGracefulRestart(stalePathTime);
 644                 } catch (TException | BgpRouterException e) {
 645                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 646                 }
 647             }
 648         }
 649
 650         @Override
 651         protected GracefulRestartReactor getDataTreeChangeListener() {
 652             return GracefulRestartReactor.this;
 653         }
 654
 655         @Override
 656         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 657             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 658         }
 659
 660         @Override
 661         protected void remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 662             if (!isBGPEntityOwner()) {
 663                 return;
 664             }
 665             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 666             synchronized (BgpConfigurationManager.this) {
 667                 BgpRouter br = getClient(YANG_OBJ);
 668                 if (br == null) {
 669                     LOG.error("{} Unable to delete stale-path time; {} {}", YANG_OBJ,
 670                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 671                     return;
 672                 }
 673                 try {
 674                     br.delGracefulRestart();
 675                 } catch (TException | BgpRouterException e) {
 676                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 677                 }
 678             }
 679         }
 680
 681         @Override
 682         protected void update(InstanceIdentifier<GracefulRestart> iid,
 683                 GracefulRestart oldval, GracefulRestart newval) {
 684             if (!isBGPEntityOwner()) {
 685                 return;
 686             }
 687             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 688             synchronized (BgpConfigurationManager.this) {
 689                 int stalePathTime = newval.getStalepathTime().intValue();
 690                 BgpRouter br = getClient(YANG_OBJ);
 691                 if (br == null) {
 692                     LOG.error("{} Unable to update stale-path time to {}; {} {}", YANG_OBJ, stalePathTime,
 693                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 694                     return;
 695                 }
 696                 try {
 697                     br.addGracefulRestart(stalePathTime);
 698                 } catch (TException | BgpRouterException e) {
 699                     LOG.error("{} update received exception; {}", YANG_OBJ, ADD_WARN, e);
 700                 }
 701             }
 702         }
 703     }
 704
 705     public class LoggingReactor
 706             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 707             implements ClusteredDataTreeChangeListener<Logging> {
 708
 709         private static final String YANG_OBJ = "logging ";
 710
 711         public LoggingReactor() {
 712             super(Logging.class, LoggingReactor.class);
 713         }
 714
 715         @Override
 716         protected void add(InstanceIdentifier<Logging> iid, Logging val) {
 717             if (!isBGPEntityOwner()) {
 718                 return;
 719             }
 720             synchronized (BgpConfigurationManager.this) {
 721                 BgpRouter br = getClient(YANG_OBJ);
 722                 if (br == null) {
 723                     LOG.error("{} Unable to add logging for qbgp; {} {}", YANG_OBJ,
 724                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 725                     return;
 726                 }
 727                 try {
 728                     br.setLogging(val.getFile(), val.getLevel());
 729                 } catch (TException | BgpRouterException e) {
 730                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 731                 }
 732             }
 733         }
 734
 735         @Override
 736         protected LoggingReactor getDataTreeChangeListener() {
 737             return LoggingReactor.this;
 738         }
 739
 740         @Override
 741         protected InstanceIdentifier<Logging> getWildCardPath() {
 742             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 743         }
 744
 745         @Override
 746         protected void remove(InstanceIdentifier<Logging> iid, Logging val) {
 747             if (!isBGPEntityOwner()) {
 748                 return;
 749             }
 750             LOG.debug("received remove Logging config val {}", val.getLevel());
 751             synchronized (BgpConfigurationManager.this) {
 752                 BgpRouter br = getClient(YANG_OBJ);
 753                 if (br == null) {
 754                     LOG.error("{} Unable to remove logging for qbgp; {} {}", YANG_OBJ,
 755                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 756                     return;
 757                 }
 758                 try {
 759                     br.setLogging(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
 760                 } catch (TException | BgpRouterException e) {
 761                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 762                 }
 763             }
 764         }
 765
 766         @Override
 767         protected void update(InstanceIdentifier<Logging> iid,
 768                 Logging oldval, Logging newval) {
 769             if (!isBGPEntityOwner()) {
 770                 return;
 771             }
 772             synchronized (BgpConfigurationManager.this) {
 773                 BgpRouter br = getClient(YANG_OBJ);
 774                 if (br == null) {
 775                     LOG.error("{} Unable to update logging for qbgp; {} {}", YANG_OBJ,
 776                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 777                     return;
 778                 }
 779                 try {
 780                     br.setLogging(newval.getFile(), newval.getLevel());
 781                 } catch (TException | BgpRouterException e) {
 782                     LOG.error("{} newval received exception; {}", YANG_OBJ, ADD_WARN, e);
 783                 }
 784             }
 785         }
 786     }
 787
 788     public class NeighborsReactor
 789             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 790             implements ClusteredDataTreeChangeListener<Neighbors> {
 791
 792         private static final String YANG_OBJ = "neighbors ";
 793
 794         public NeighborsReactor() {
 795             super(Neighbors.class, NeighborsReactor.class);
 796         }
 797
 798         @Override
 799         protected void add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 800             if (!isBGPEntityOwner()) {
 801                 return;
 802             }
 803             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 804             synchronized (BgpConfigurationManager.this) {
 805                 String peerIp = val.getAddress().getValue();
 806                 long as = val.getRemoteAs();
 807                 final String md5Secret = extractMd5Secret(val);
 808                 BgpRouter br = getClient(YANG_OBJ);
 809                 if (br == null) {
 810                     LOG.debug("{} Unable to process add for peer {} as {}; {} {}", YANG_OBJ, peerIp, as,
 811                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 812                     return;
 813                 }
 814                 try {
 815                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 816                     br.addNeighbor(peerIp, as, md5Secret);
 817
 818                 } catch (TException | BgpRouterException e) {
 819                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 820                 }
 821             }
 822         }
 823
 824         @Override
 825         protected NeighborsReactor getDataTreeChangeListener() {
 826             return NeighborsReactor.this;
 827         }
 828
 829         @Override
 830         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 831             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class);
 832         }
 833
 834         @Override
 835         protected void remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 836             if (!isBGPEntityOwner()) {
 837                 return;
 838             }
 839             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 840             synchronized (BgpConfigurationManager.this) {
 841                 String peerIp = val.getAddress().getValue();
 842                 BgpRouter br = getClient(YANG_OBJ);
 843                 if (br == null) {
 844                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 845                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 846                     return;
 847                 }
 848                 try {
 849                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 850                     br.delNeighbor(peerIp);
 851                 } catch (TException | BgpRouterException e) {
 852                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 853                 }
 854
 855                 final BgpAlarms bgpAlarms = getBgpAlarms();
 856                 if (bgpAlarms != null) {
 857                     bgpAlarms.clearBgpNbrDownAlarm(peerIp);
 858                 }
 859             }
 860         }
 861
 862         @Override
 863         protected void update(InstanceIdentifier<Neighbors> iid,
 864                 Neighbors oldval, Neighbors newval) {
 865             if (!isBGPEntityOwner()) {
 866                 return;
 867             }
 868             //purposefully nothing to do.
 869         }
 870     }
 871
 872     public class EbgpMultihopReactor
 873             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 874             implements ClusteredDataTreeChangeListener<EbgpMultihop> {
 875
 876         private static final String YANG_OBJ = "ebgp-multihop ";
 877
 878         public EbgpMultihopReactor() {
 879             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 880         }
 881
 882         @Override
 883         protected void add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 884             if (!isBGPEntityOwner()) {
 885                 return;
 886             }
 887             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 888             synchronized (BgpConfigurationManager.this) {
 889                 String peerIp = val.getPeerIp().getValue();
 890                 BgpRouter br = getClient(YANG_OBJ);
 891                 if (br == null) {
 892                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
 893                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 894                     return;
 895                 }
 896                 try {
 897                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 898                 } catch (TException | BgpRouterException e) {
 899                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 900                 }
 901             }
 902         }
 903
 904         @Override
 905         protected EbgpMultihopReactor getDataTreeChangeListener() {
 906             return EbgpMultihopReactor.this;
 907         }
 908
 909         @Override
 910         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
 911             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(EbgpMultihop.class);
 912         }
 913
 914         @Override
 915         protected void remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 916             if (!isBGPEntityOwner()) {
 917                 return;
 918             }
 919             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 920             synchronized (BgpConfigurationManager.this) {
 921                 String peerIp = val.getPeerIp().getValue();
 922                 BgpRouter br = getClient(YANG_OBJ);
 923                 if (br == null) {
 924                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 925                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 926                     return;
 927                 }
 928                 try {
 929                     br.delEbgpMultihop(peerIp);
 930                 } catch (TException | BgpRouterException e) {
 931                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 932                 }
 933             }
 934         }
 935
 936         @Override
 937         protected void update(InstanceIdentifier<EbgpMultihop> iid,
 938                 EbgpMultihop oldval, EbgpMultihop newval) {
 939             if (!isBGPEntityOwner()) {
 940                 return;
 941             }
 942             LOG.error(YANG_OBJ + UPD_WARN);
 943         }
 944     }
 945
 946     public class UpdateSourceReactor
 947             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
 948             implements ClusteredDataTreeChangeListener<UpdateSource> {
 949
 950         private static final String YANG_OBJ = "update-source ";
 951
 952         public UpdateSourceReactor() {
 953             super(UpdateSource.class, UpdateSourceReactor.class);
 954         }
 955
 956         @Override
 957         protected void add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 958             if (!isBGPEntityOwner()) {
 959                 return;
 960             }
 961             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
 962             synchronized (BgpConfigurationManager.this) {
 963                 String peerIp = val.getPeerIp().getValue();
 964                 BgpRouter br = getClient(YANG_OBJ);
 965                 if (br == null) {
 966                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
 967                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 968                     return;
 969                 }
 970                 try {
 971                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
 972                 } catch (TException | BgpRouterException e) {
 973                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 974                 }
 975             }
 976         }
 977
 978         @Override
 979         protected UpdateSourceReactor getDataTreeChangeListener() {
 980             return UpdateSourceReactor.this;
 981         }
 982
 983         @Override
 984         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
 985             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(UpdateSource.class);
 986         }
 987
 988         @Override
 989         protected void remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 990             if (!isBGPEntityOwner()) {
 991                 return;
 992             }
 993             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
 994             synchronized (BgpConfigurationManager.this) {
 995                 String peerIp = val.getPeerIp().getValue();
 996                 BgpRouter br = getClient(YANG_OBJ);
 997                 if (br == null) {
 998                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 999                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1000                     return;
1001                 }
1002                 try {
1003                     br.delUpdateSource(peerIp);
1004                 } catch (TException | BgpRouterException e) {
1005                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1006                 }
1007             }
1008         }
1009
1010         @Override
1011         protected void update(InstanceIdentifier<UpdateSource> iid,
1012                 UpdateSource oldval, UpdateSource newval) {
1013             if (!isBGPEntityOwner()) {
1014                 return;
1015             }
1016             LOG.error(YANG_OBJ + UPD_WARN);
1017         }
1018     }
1019
1020     public class AddressFamiliesReactor
1021             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
1022             implements ClusteredDataTreeChangeListener<AddressFamilies> {
1023
1024         private static final String YANG_OBJ = "address-families ";
1025
1026         public AddressFamiliesReactor() {
1027             super(AddressFamilies.class, AddressFamiliesReactor.class);
1028         }
1029
1030         @Override
1031         protected void add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1032             if (!isBGPEntityOwner()) {
1033                 return;
1034             }
1035             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
1036             synchronized (BgpConfigurationManager.this) {
1037                 String peerIp = val.getPeerIp().getValue();
1038                 BgpRouter br = getClient(YANG_OBJ);
1039                 if (br == null) {
1040                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1041                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1042                     return;
1043                 }
1044                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1045                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1046                 try {
1047                     br.addAddressFamily(peerIp, afi, safi);
1048                 } catch (TException | BgpRouterException e) {
1049                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1050                 }
1051             }
1052         }
1053
1054         @Override
1055         protected AddressFamiliesReactor getDataTreeChangeListener() {
1056             return AddressFamiliesReactor.this;
1057         }
1058
1059         @Override
1060         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
1061             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(AddressFamilies.class);
1062         }
1063
1064         @Override
1065         protected void remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1066             if (!isBGPEntityOwner()) {
1067                 return;
1068             }
1069             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
1070             synchronized (BgpConfigurationManager.this) {
1071                 String peerIp = val.getPeerIp().getValue();
1072                 BgpRouter br = getClient(YANG_OBJ);
1073                 if (br == null) {
1074                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1075                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1076                     return;
1077                 }
1078                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1079                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1080                 try {
1081                     br.delAddressFamily(peerIp, afi, safi);
1082                 } catch (TException | BgpRouterException e) {
1083                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1084                 }
1085             }
1086         }
1087
1088         @Override
1089         protected void update(InstanceIdentifier<AddressFamilies> iid,
1090                 AddressFamilies oldval, AddressFamilies newval) {
1091             if (!isBGPEntityOwner()) {
1092                 return;
1093             }
1094             LOG.error(YANG_OBJ + UPD_WARN);
1095         }
1096     }
1097
1098     public class NetworksReactor
1099             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
1100             implements ClusteredDataTreeChangeListener<Networks> {
1101
1102         private static final String YANG_OBJ = "networks ";
1103
1104         public NetworksReactor() {
1105             super(Networks.class, NetworksReactor.class);
1106         }
1107
1108         @Override
1109         public NetworksReactor getDataTreeChangeListener() {
1110             return NetworksReactor.this;
1111         }
1112
1113         @Override
1114         protected void add(InstanceIdentifier<Networks> iid, Networks val) {
1115             if (!isBGPEntityOwner()) {
1116                 return;
1117             }
1118             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1119             synchronized (BgpConfigurationManager.this) {
1120                 String rd = val.getRd();
1121                 String pfxlen = val.getPrefixLen();
1122                 String nh = val.getNexthop().getValue();
1123                 BgpRouter br = getClient(YANG_OBJ);
1124                 if (br == null) {
1125                     LOG.debug("{} Unable to process add for rd {} prefix {} nexthop {}; {} {}", YANG_OBJ, rd, pfxlen,
1126                             nh, BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1127                     return;
1128                 }
1129                 Long label = val.getLabel();
1130                 int lbl = label == null ? qbgpConstants.LBL_NO_LABEL
1131                         : label.intValue();
1132                 int l3vni = val.getL3vni() == null ? qbgpConstants.LBL_NO_LABEL
1133                         : val.getL3vni().intValue();
1134                 int l2vni = val.getL2vni() == null ? qbgpConstants.LBL_NO_LABEL
1135                         : val.getL2vni().intValue();
1136
1137                 BgpControlPlaneType protocolType = val.getBgpControlPlaneType();
1138                 int ethernetTag = val.getEthtag().intValue();
1139                 String esi = val.getEsi();
1140                 String macaddress = val.getMacaddress();
1141                 EncapType encapType = val.getEncapType();
1142                 String routerMac = val.getRoutermac();
1143
1144                 try {
1145                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni, BgpUtil.convertToThriftProtocolType(protocolType),
1146                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1147                 } catch (TException | BgpRouterException e) {
1148                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1149                 }
1150             }
1151         }
1152
1153         @Override
1154         protected InstanceIdentifier<Networks> getWildCardPath() {
1155             return InstanceIdentifier.create(Bgp.class).child(Networks.class);
1156         }
1157
1158         @Override
1159         protected void remove(InstanceIdentifier<Networks> iid, Networks val) {
1160             if (!isBGPEntityOwner()) {
1161                 return;
1162             }
1163             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1164             synchronized (BgpConfigurationManager.this) {
1165                 String rd = val.getRd();
1166                 String pfxlen = val.getPrefixLen();
1167                 BgpRouter br = getClient(YANG_OBJ);
1168                 if (br == null) {
1169                     LOG.debug("{} Unable to process remove for rd {} prefix {}; {} {}", YANG_OBJ, rd, pfxlen,
1170                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1171                     return;
1172                 }
1173                 Long label = val.getLabel();
1174                 int lbl = label == null ? 0 : label.intValue();
1175                 if (rd == null && lbl > 0) {
1176                     //LU prefix is being deleted.
1177                     rd = Integer.toString(lbl);
1178                 }
1179                 try {
1180                     br.delPrefix(rd, pfxlen);
1181                 } catch (TException | BgpRouterException e) {
1182                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1183                 }
1184             }
1185         }
1186
1187         /**get the value AFI from a prefix as "x.x.x.x/x".
1188          *
1189          * @param pfxlen the prefix to get an afi
1190          * @return the afi value as you are need
1191          */
1192         public  int  testValueAFI(String pfxlen) {
1193             int afiNew = af_afi.AFI_IP.getValue();
1194             try {
1195                 String ipOnly = pfxlen.substring(0, pfxlen.lastIndexOf("/"));
1196                 java.net.Inet6Address.getByName(ipOnly);
1197                 afiNew = af_afi.AFI_IPV6.getValue();
1198             } catch (java.net.UnknownHostException e) {
1199                 //ce n'est pas de l'ipv6
1200             }
1201             return afiNew;
1202         }
1203
1204
1205         @Override
1206         protected void update(final InstanceIdentifier<Networks> iid,
1207                 final Networks oldval, final Networks newval) {
1208             if (!isBGPEntityOwner()) {
1209                 return;
1210             }
1211             if (oldval.equals(newval)) {
1212                 //Update: OLD and New values are same, no need to trigger remove/add.
1213                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1214                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1215                 return;
1216             }
1217             LOG.debug("update networks old val RD: {}, Prefix: {}, Label: {}, NH: {} "
1218                             + "new val RD: {}, Prefix: {}, Label: {}, NH: {}",
1219                     oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop(),
1220                     newval.getRd(), newval.getPrefixLen(), newval.getLabel(), newval.getNexthop());
1221             remove(iid, oldval);
1222             add(iid, newval);
1223         }
1224     }
1225
1226     static Timer timer = new Timer();
1227
1228     public class VrfsReactor
1229             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1230             implements ClusteredDataTreeChangeListener<Vrfs> {
1231
1232         private static final String YANG_OBJ = "vrfs ";
1233
1234         public VrfsReactor() {
1235             super(Vrfs.class, VrfsReactor.class);
1236         }
1237
1238         @Override
1239         protected void add(InstanceIdentifier<Vrfs> iid, Vrfs vrfs) {
1240             if (!isBGPEntityOwner()) {
1241                 return;
1242             }
1243             LOG.debug("received add Vrfs config value {}", vrfs.getRd());
1244             synchronized (BgpConfigurationManager.this) {
1245                 String rd = vrfs.getRd();
1246                 BgpRouter br = getClient(YANG_OBJ);
1247                 if (br == null) {
1248                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1249                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1250                     return;
1251                 }
1252                 try {
1253                     List<AddressFamiliesVrf> vrfAddrFamilyList = vrfs.getAddressFamiliesVrf();
1254                     for (AddressFamiliesVrf vrfAddrFamily : vrfAddrFamilyList) {
1255                         /*add to br the new vrfs arguments*/
1256                         br.addVrf(BgpUtil.getLayerType(vrfAddrFamily), rd, vrfs.getImportRts(), vrfs.getExportRts()
1257                         );
1258                     }
1259                     /*add to br the vrfs contained in mapNewAdFamily*/
1260                     List<AddressFamiliesVrf> vrfAddrFamilyListFromMap = mapNewAdFamily.get(rd);
1261                     if (vrfAddrFamilyListFromMap == null) {
1262                         return;
1263                     }
1264
1265                     for (AddressFamiliesVrf adf : vrfAddrFamilyListFromMap) {
1266                         if (vrfAddrFamilyList.contains(adf)) {
1267                             mapNewAdFamily.remove(rd);
1268                         } else  if (adf != null) {
1269
1270                             br.addVrf(BgpUtil.getLayerType(adf), rd, vrfs.getImportRts(), vrfs.getExportRts()
1271                             );
1272                             // remove AddressFamiliesVrf which was already added to BGP
1273                             vrfAddrFamilyListFromMap.remove(adf);
1274                             if (vrfAddrFamilyListFromMap.isEmpty()) {
1275                                 // remove Vrf entry from temp mapNewAdFamily if all its AddressFamiliesVrf was
1276                                 // added to BGP
1277                                 mapNewAdFamily.remove(rd);
1278                             }
1279                         }
1280                     }
1281                 } catch (TException | BgpRouterException e) {
1282                     LOG.error("{} get {}, Add received exception", YANG_OBJ, ADD_WARN, e);
1283                 }
1284             }
1285         }
1286
1287         @Override
1288         protected VrfsReactor getDataTreeChangeListener() {
1289             return VrfsReactor.this;
1290         }
1291
1292         @Override
1293         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1294             return InstanceIdentifier.create(Bgp.class).child(Vrfs.class);
1295         }
1296
1297         @Override
1298         protected void remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1299             if (!isBGPEntityOwner()) {
1300                 return;
1301             }
1302             LOG.debug("received remove Vrfs config val {}", val.getRd());
1303             synchronized (BgpConfigurationManager.this) {
1304                 String rd = val.getRd();
1305                 BgpRouter br = getClient(YANG_OBJ);
1306                 if (br == null) {
1307                     LOG.debug("{} Unable to process remove for rd {}; {} {}", YANG_OBJ, rd,
1308                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1309                     return;
1310                 }
1311                 try {
1312                     List<AddressFamiliesVrf> adf = mapNewAdFamily.get(rd);
1313                     adf = adf != null ? adf : new ArrayList<>();
1314                     for (AddressFamiliesVrf s : val.getAddressFamiliesVrf()) {
1315                         br.delVrf(rd, s.getAfi(), s.getSafi());
1316                         adf.remove(s);// remove in the map the vrf in waiting for advertise quagga
1317                     }
1318                     if (adf.isEmpty()) {
1319                         mapNewAdFamily.remove(rd);
1320                     }
1321                 } catch (TException | BgpRouterException e) {
1322                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1323                 }
1324             }
1325         }
1326
1327         @Override
1328         protected void update(InstanceIdentifier<Vrfs> iid,
1329                 Vrfs oldval, Vrfs newval) {
1330             if (oldval != null && newval != null) {
1331                 LOG.debug("received update Vrfs config val {}, VRFS: Update getting triggered for VRFS rd {}",
1332                         newval.getRd(), oldval.getRd());
1333             } else {
1334                 LOG.debug("received update Vrfs config val {}, from old vrf {}",
1335                         newval, oldval);
1336             }
1337             if (!isBGPEntityOwner()) {
1338                 return;
1339             }
1340
1341             List<AddressFamiliesVrf> adFamilyVrfToDel = new ArrayList<>();
1342             List<AddressFamiliesVrf> adFamilyVrfToAdd = new ArrayList<>();
1343             List<AddressFamiliesVrf> oldlistAdFamilies = new ArrayList<>();
1344             List<AddressFamiliesVrf> newlistAdFamilies = new ArrayList<>();
1345             if (oldval != null) {
1346                 oldlistAdFamilies = oldval.getAddressFamiliesVrf() == null
1347                         ? new ArrayList<>() : oldval.getAddressFamiliesVrf();
1348             }
1349             if (newval != null) {
1350                 newlistAdFamilies = newval.getAddressFamiliesVrf() == null
1351                         ? new ArrayList<>() : newval.getAddressFamiliesVrf();
1352             }
1353             /*find old AddressFamily to remove from new configuration*/
1354             for (AddressFamiliesVrf adVrf : oldlistAdFamilies) {
1355                 if (!newlistAdFamilies.contains(adVrf)) {
1356                     adFamilyVrfToDel.add(adVrf);
1357                 }
1358             }
1359             /*find new AddressFamily to add to unexisting configuration*/
1360             for (AddressFamiliesVrf adVrf : newlistAdFamilies) {
1361                 if (!oldlistAdFamilies.contains(adVrf)) {
1362                     adFamilyVrfToAdd.add(adVrf);
1363                 }
1364             }
1365             String rd = newval != null ? newval.getRd() : null;
1366             if (rd != null) {
1367                 BgpRouter br = getClient(YANG_OBJ);
1368                 if (br == null) {
1369                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1370                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1371                     return;
1372                 }
1373
1374                 for (AddressFamiliesVrf adfvrf : adFamilyVrfToAdd) {
1375                     try {
1376                         LOG.debug("call addVRf rd {} afi {} safi {}", rd, adfvrf.getAfi(), adfvrf.getSafi());
1377                         br.addVrf(BgpUtil.getLayerType(adfvrf), rd, newval.getImportRts(),
1378                                 newval.getExportRts());
1379                     } catch (TException | BgpRouterException e) {
1380                         LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1381                     }
1382                 }
1383
1384                 for (AddressFamiliesVrf adfToDel : adFamilyVrfToDel) {
1385                     try {
1386                         LOG.debug("call delVRf rd {} afi {} safi {}", rd, adfToDel.getAfi(), adfToDel.getSafi());
1387                         br.delVrf(rd, adfToDel.getAfi(), adfToDel.getSafi());
1388                     } catch (TException | BgpRouterException e) {
1389                         LOG.error("{} delVrf received exception; {}", YANG_OBJ, ADD_WARN, e);
1390                     }
1391                 }
1392             }
1393         }
1394     }
1395
1396     protected void activateMIP() {
1397         try {
1398             LOG.trace("BgpReactor: Executing MIP Activate command");
1399             Runtime.getRuntime().exec("cluster ip -a sdnc_bgp_mip");
1400             Runtime.getRuntime().exec("cluster ip -a sdnc_os_mip");
1401             LOG.trace("bgpMIP Activated");
1402
1403         } catch (IOException io) {
1404             LOG.error("IO Exception got while activating mip: {}", io.getMessage());
1405         }
1406     }
1407
1408     public class BgpReactor
1409             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1410             implements ClusteredDataTreeChangeListener<Bgp> {
1411
1412         private static final String YANG_OBJ = "Bgp ";
1413
1414         public BgpReactor() {
1415             super(Bgp.class, BgpReactor.class);
1416         }
1417
1418
1419         @Override
1420         protected void add(InstanceIdentifier<Bgp> iid, Bgp val) {
1421             LOG.debug("received add Bgp config");
1422
1423             try {
1424                 initer.await();
1425             } catch (InterruptedException e) {
1426                 // Ignored
1427             }
1428             synchronized (BgpConfigurationManager.this) {
1429                 config = val;
1430                 if (!isBGPEntityOwner()) {
1431                     return;
1432                 }
1433                 activateMIP();
1434             }
1435         }
1436
1437         @Override
1438         protected BgpReactor getDataTreeChangeListener() {
1439             return BgpReactor.this;
1440         }
1441
1442         @Override
1443         protected InstanceIdentifier<Bgp> getWildCardPath() {
1444             return InstanceIdentifier.create(Bgp.class);
1445         }
1446
1447         @Override
1448         protected void remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1449             if (!isBGPEntityOwner()) {
1450                 return;
1451             }
1452             LOG.debug("received remove Bgp config");
1453
1454             config = null;
1455         }
1456
1457         @Override
1458         protected void update(InstanceIdentifier<Bgp> iid,
1459                 Bgp oldval, Bgp newval) {
1460             if (!isBGPEntityOwner()) {
1461                 return;
1462             }
1463
1464             config = newval;
1465         }
1466     }
1467
1468     @SuppressWarnings("deprecation")
1469     public class MultipathReactor
1470             extends AsyncDataTreeChangeListenerBase<Multipath, MultipathReactor>
1471             implements ClusteredDataTreeChangeListener<Multipath> {
1472
1473         private static final String YANG_OBJ = "multipath ";
1474
1475         public MultipathReactor() {
1476             super(Multipath.class, MultipathReactor.class);
1477         }
1478
1479
1480         @Override
1481         protected MultipathReactor getDataTreeChangeListener() {
1482             return MultipathReactor.this;
1483         }
1484
1485         @Override
1486         protected InstanceIdentifier<Multipath> getWildCardPath() {
1487             return InstanceIdentifier.create(Bgp.class).child(Multipath.class);
1488         }
1489
1490         @Override
1491         protected void remove(InstanceIdentifier<Multipath> iid, Multipath val) {
1492             executor.execute(new MultipathStatusChange(val));
1493         }
1494
1495         @Override
1496         protected void update(InstanceIdentifier<Multipath> iid, Multipath oldval, Multipath newval) {
1497             executor.execute(new MultipathStatusChange(newval));
1498         }
1499
1500         @Override
1501         protected void add(InstanceIdentifier<Multipath> key, Multipath dataObjectModification) {
1502             executor.execute(new MultipathStatusChange(dataObjectModification));
1503         }
1504
1505         class MultipathStatusChange implements Runnable {
1506
1507             Multipath multipath;
1508
1509             MultipathStatusChange(Multipath multipath) {
1510                 this.multipath = multipath;
1511             }
1512
1513             @Override
1514             public void run() {
1515                 if (isBGPEntityOwner()) {
1516                     synchronized (BgpConfigurationManager.this) {
1517
1518                         BgpRouter br = getClient(YANG_OBJ);
1519
1520                         if (br != null) {
1521                             af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1522                             af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1523
1524                             try {
1525                                 if (multipath.isMultipathEnabled()) {
1526                                     br.enableMultipath(afi, safi);
1527                                 } else {
1528                                     br.disableMultipath(afi, safi);
1529                                 }
1530                             } catch (TException | BgpRouterException e) {
1531                                 LOG.error("{} received exception", YANG_OBJ, e);
1532                             }
1533                         }
1534                     }
1535                 }
1536             }
1537
1538         }
1539
1540         @Override
1541         public void close() {
1542             super.close();
1543         }
1544     }
1545
1546     @SuppressWarnings("deprecation")
1547     public class VrfMaxpathReactor
1548             extends AsyncDataTreeChangeListenerBase<VrfMaxpath, VrfMaxpathReactor>
1549             implements ClusteredDataTreeChangeListener<VrfMaxpath> {
1550
1551         private static final String YANG_OBJ = "vrfMaxpath ";
1552
1553         public VrfMaxpathReactor() {
1554             super(VrfMaxpath.class, VrfMaxpathReactor.class);
1555         }
1556
1557
1558         @Override
1559         protected VrfMaxpathReactor getDataTreeChangeListener() {
1560             return VrfMaxpathReactor.this;
1561         }
1562
1563         @Override
1564         protected InstanceIdentifier<VrfMaxpath> getWildCardPath() {
1565             return InstanceIdentifier.create(Bgp.class).child(VrfMaxpath.class);
1566         }
1567
1568         class VrfMaxPathConfigurator implements Runnable {
1569
1570             VrfMaxpath vrfMaxpathVal;
1571
1572             VrfMaxPathConfigurator(VrfMaxpath vrfMaxPathVal) {
1573                 this.vrfMaxpathVal = vrfMaxPathVal;
1574             }
1575
1576             @Override
1577             public void run() {
1578                 if (isBGPEntityOwner()) {
1579                     synchronized (BgpConfigurationManager.this) {
1580                         BgpRouter br = getClient(YANG_OBJ);
1581                         if (br != null) {
1582                             try {
1583                                 br.multipaths(vrfMaxpathVal.getRd(), vrfMaxpathVal.getMaxpaths());
1584                                 LOG.debug("Maxpath for vrf {} is {}", vrfMaxpathVal.getRd(),
1585                                         vrfMaxpathVal.getMaxpaths());
1586                             } catch (TException | BgpRouterException e) {
1587                                 LOG.error("{} received exception", YANG_OBJ, e);
1588                             }
1589                         }
1590                     }
1591                 }
1592             }
1593         }
1594
1595         @Override
1596         protected void remove(InstanceIdentifier<VrfMaxpath> iid, VrfMaxpath vrfMaxPathVal) {
1597             executor.execute(new VrfMaxPathConfigurator(vrfMaxPathVal));
1598         }
1599
1600         @Override
1601         protected void update(InstanceIdentifier<VrfMaxpath> iid,
1602                               VrfMaxpath oldval, VrfMaxpath newval) {
1603             if (!Objects.equals(oldval.getMaxpaths(), newval.getMaxpaths())) {
1604                 executor.execute(new VrfMaxPathConfigurator(newval));
1605             }
1606         }
1607
1608         @Override
1609         protected void add(InstanceIdentifier<VrfMaxpath> instanceIdentifier, VrfMaxpath vrfMaxpathVal) {
1610             executor.execute(new VrfMaxPathConfigurator(vrfMaxpathVal));
1611         }
1612
1613         @Override
1614         public void close() {
1615             super.close();
1616         }
1617     }
1618
1619     public boolean isIpAvailable(String odlip) {
1620
1621         try {
1622             if (odlip != null) {
1623                 if ("127.0.0.1".equals(odlip)) {
1624                     return true;
1625                 }
1626                 Enumeration<NetworkInterface> networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces();
1627                 while (networkInterfaceEnumeration.hasMoreElements()) {
1628                     NetworkInterface networkInterface = networkInterfaceEnumeration.nextElement();
1629                     Enumeration<InetAddress>  inetAddressEnumeration = networkInterface.getInetAddresses();
1630                     while (inetAddressEnumeration.hasMoreElements()) {
1631                         InetAddress inetAddress = inetAddressEnumeration.nextElement();
1632                         if (odlip.equals(inetAddress.getHostAddress())) {
1633                             return true;
1634                         }
1635                     }
1636                 }
1637             }
1638         } catch (SocketException e) {
1639             // Ignored?
1640         }
1641         return false;
1642     }
1643
1644     public long getStalePathtime(int defValue, AsId asId) {
1645         long spt = 0;
1646         try {
1647             spt = getConfig().getGracefulRestart().getStalepathTime();
1648         } catch (NullPointerException e) {
1649             try {
1650                 spt = asId.getStalepathTime();
1651                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1652             } catch (NullPointerException ignore) {
1653                 LOG.trace("BGP AS id is not set using graceful");
1654                 spt = defValue;
1655             }
1656         }
1657         if (spt == 0) {
1658             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1659             spt = defValue;
1660         }
1661         return spt;
1662     }
1663
1664     public static boolean isValidConfigBgpHostPort(String bgpHost, int bgpPort) {
1665         if (!bgpHost.equals(DEF_CHOST)) {
1666             return true;
1667         } else {
1668             return false;
1669         }
1670     }
1671
1672     public synchronized void bgpRestarted() {
1673         /*
1674          * If there a thread which in the process of stale cleanup, cancel it
1675          * and start a new thread (to avoid processing same again).
1676          */
1677         if (previousReplayJobInProgress()) {
1678             cancelPreviousReplayJob();
1679         }
1680         Runnable task = () -> {
1681             try {
1682                 LOG.info("running bgp replay task ");
1683                 if (get() == null) {
1684                     String host = getConfigHost();
1685                     int port = getConfigPort();
1686                     LOG.info("connecting  to bgp host {} ", host);
1687                     bgpRouter.connect(host, port);
1688                     LOG.info("no config to push in bgp replay task ");
1689                     return;
1690                 }
1691                 setStaleStartTime(System.currentTimeMillis());
1692                 LOG.info("started creating stale fibDSWriter  map ");
1693                 createStaleFibMap();
1694                 setStaleEndTime(System.currentTimeMillis());
1695                 LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
1696                 LOG.info("started bgp config replay ");
1697                 setCfgReplayStartTime(System.currentTimeMillis());
1698                 boolean replaySucceded = replay();
1699                 setCfgReplayEndTime(System.currentTimeMillis());
1700                 LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1701                 if (replaySucceded) {
1702                     LOG.info("starting the stale cleanup timer");
1703                     long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1704                     setStaleCleanupTime(routeSyncTime);
1705                     routeCleanupFuture = executor.schedule(new RouteCleanup(), routeSyncTime, TimeUnit.SECONDS);
1706                 } else {
1707                     staledFibEntriesMap.clear();
1708                 }
1709             } catch (InterruptedException | TimeoutException | ExecutionException eCancel) {
1710                 LOG.error("Stale Cleanup Task Cancelled", eCancel);
1711             }
1712         };
1713         lastReplayJobFt = executor.submit(task);
1714     }
1715
1716     private boolean previousReplayJobInProgress() {
1717         return lastReplayJobFt != null && !lastReplayJobFt.isDone();
1718     }
1719
1720     private void cancelPreviousReplayJob() {
1721         try {
1722             LOG.error("cancelling already running bgp replay task");
1723             if (lastReplayJobFt != null) {
1724                 lastReplayJobFt.cancel(true);
1725                 lastReplayJobFt = null;
1726                 staledFibEntriesMap.clear();
1727             }
1728             if (routeCleanupFuture != null) {
1729                 routeCleanupFuture.cancel(true);
1730                 routeCleanupFuture = null;
1731                 staledFibEntriesMap.clear();
1732             }
1733             Thread.sleep(2000);
1734         } catch (InterruptedException e) {
1735             LOG.error("Failed to cancel previous replay job ", e);
1736         }
1737     }
1738
1739     private void doRouteSync() {
1740         LOG.error("Starting BGP route sync");
1741         try {
1742             bgpRouter.initRibSync(bgpSyncHandle);
1743         } catch (BgpRouterException e) {
1744             LOG.error("Route sync aborted, exception when initializing", e);
1745             return;
1746         }
1747         while (bgpSyncHandle.getState() != BgpSyncHandle.DONE) {
1748             for (af_afi afi : af_afi.values()) {
1749                 Routes routes = null;
1750                 try {
1751                     routes = bgpRouter.doRibSync(bgpSyncHandle, afi);
1752                 } catch (TException | BgpRouterException e) {
1753                     LOG.error("Route sync aborted, exception when syncing", e);
1754                     return;
1755                 }
1756                 Iterator<Update> updates = routes.getUpdatesIterator();
1757                 while (updates.hasNext()) {
1758                     Update update = updates.next();
1759                     String rd = update.getRd();
1760                     String nexthop = update.getNexthop();
1761
1762                     // TODO: decide correct label here
1763                     int label = update.getL3label();
1764                     int l2label = update.getL2label();
1765
1766                     String prefix = update.getPrefix();
1767                     int plen = update.getPrefixlen();
1768
1769
1770                     // TODO: protocol type will not be available in "update"
1771                     // use "rd" to query vrf table and obtain the protocol_type.
1772                     // Currently using PROTOCOL_EVPN as default.
1773                     onUpdatePushRoute(
1774                            protocol_type.PROTOCOL_EVPN,
1775                            rd,
1776                            prefix,
1777                            plen,
1778                            nexthop,
1779                            update.getMacaddress(),
1780                            label,
1781                            l2label,
1782                            update.getRoutermac(),
1783                            afi);
1784                 }
1785             }
1786         }
1787         try {
1788             LOG.error("Ending BGP route-sync");
1789             bgpRouter.endRibSync(bgpSyncHandle);
1790         } catch (BgpRouterException e) {
1791             // Ignored?
1792         }
1793     }
1794
1795     public void addTepToElanDS(String rd, String tepIp, String mac, Long l2vni) {
1796         boolean needUpdate = addToRt2TepMap(rd, tepIp, mac, l2vni);
1797         if (needUpdate) {
1798             LOG.info("Adding tepIp {} with RD {} to ELan DS", tepIp, rd);
1799             bgpUtil.addTepToElanInstance(rd, tepIp);
1800         } else {
1801             LOG.debug("Skipping the Elan update for RT2 from tep {} rd {}", tepIp, rd);
1802         }
1803     }
1804
1805     public void deleteTepfromElanDS(String rd, String tepIp, String mac) {
1806         boolean needUpdate = deleteFromRt2TepMap(rd, tepIp, mac);
1807         if (needUpdate) {
1808             LOG.info("Deleting tepIp {} with RD {} to ELan DS", tepIp, rd);
1809             bgpUtil.deleteTepFromElanInstance(rd, tepIp);
1810         } else {
1811             LOG.debug("Skipping the Elan update for RT2 withdraw from tep {} rd {}", tepIp, rd);
1812         }
1813     }
1814
1815     /* onUpdatePushRoute
1816      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1817      *  - Entry compare shall include NextHop, Label.
1818      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1819      *  - If entry not found, add to FIB Config DS.
1820      *  - If entry found, but either Label/NextHop doesn't match.
1821      *      - Update FIB Config DS with modified values.
1822      *      - delete from Stale Map.
1823      */
1824
1825     public void onUpdatePushRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
1826                                   String macaddress, int label, int l2label, String routermac, af_afi afi) {
1827         boolean addroute = false;
1828         boolean macupdate = false;
1829         long l3vni = 0L;
1830         VrfEntry.EncapType encapType = VrfEntry.EncapType.Mplsgre;
1831         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1832             encapType = VrfEntry.EncapType.Vxlan;
1833             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
1834             if (vpnInstanceOpDataEntry != null) {
1835                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
1836                     LOG.info("Got RT2 route for RD {} l3label {} l2label {} from tep {} with mac {} remote RD {}",
1837                             vpnInstanceOpDataEntry.getVpnInstanceName(), label, l2label, nextHop, macaddress, rd);
1838                     addTepToElanDS(rd, nextHop, macaddress, (long)l2label);
1839                     macupdate = true;
1840                 } else {
1841                     l3vni = vpnInstanceOpDataEntry.getL3vni();
1842                 }
1843             } else {
1844                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1845                 return;
1846             }
1847         }
1848
1849         if (!staledFibEntriesMap.isEmpty()) {
1850             // restart Scenario, as MAP is not empty.
1851             Map<String, Long> map = staledFibEntriesMap.get(rd);
1852             if (map != null) {
1853                 String prefixNextHop = appendNextHopToPrefix(prefix + "/" + plen, nextHop);
1854                 Long labelInStaleMap = map.get(prefixNextHop);
1855                 if (null == labelInStaleMap) {
1856                     // New Entry, which happened to be added during restart.
1857                     addroute = true;
1858                 } else {
1859                     map.remove(prefixNextHop);
1860                     if (isRouteModified(label, labelInStaleMap)) {
1861                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1862                         // Existing entry, where in Label got modified during restart
1863                         addroute = true;
1864                     }
1865                 }
1866             } else {
1867                 LOG.debug("rd {} map is null while processing prefix {} ", rd, prefix);
1868                 addroute = true;
1869             }
1870         } else {
1871             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1872             addroute = true;
1873         }
1874         if (macupdate) {
1875             LOG.info("ADD: Adding Mac Fib entry rd {} mac{} nexthop {} l2vni {}", rd, macaddress, nextHop, l2label);
1876             fibDSWriter.addMacEntryToDS(rd, macaddress, prefix, Collections.singletonList(nextHop),
1877                     encapType, l2label, routermac, RouteOrigin.BGP);
1878             LOG.info("ADD: Added Mac Fib entry rd {} prefix {} nexthop {} label {}", rd, macaddress, nextHop, l2label);
1879         } else if (addroute) {
1880             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {} afi {}",
1881                     rd, prefix, nextHop, label, afi);
1882             // TODO: modify addFibEntryToDS signature
1883             List<String> nextHopList = Collections.singletonList(nextHop);
1884             fibDSWriter.addFibEntryToDS(rd, prefix + "/" + plen, nextHopList, encapType, label, l3vni,
1885                                         routermac, RouteOrigin.BGP);
1886             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1887             String vpnName = bgpUtil.getVpnNameFromRd(rd);
1888             if (vpnName != null) {
1889                 vpnLinkService.leakRouteIfNeeded(vpnName, prefix, nextHopList, label, RouteOrigin.BGP,
1890                                                  NwConstants.ADD_FLOW);
1891             }
1892         }
1893     }
1894
1895     public void onUpdateWithdrawRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
1896             String macaddress) {
1897         long vni = 0L;
1898         boolean macupdate = false;
1899         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1900             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
1901             if (vpnInstanceOpDataEntry != null) {
1902                 vni = vpnInstanceOpDataEntry.getL3vni();
1903                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
1904                     LOG.debug("Got RT2 withdraw for RD {} {} from tep {} with mac {} remote RD {}",
1905                             vpnInstanceOpDataEntry.getVpnInstanceName(), vni, nextHop, macaddress, rd);
1906                     deleteTepfromElanDS(rd, nextHop, macaddress);
1907                     LOG.debug("For rd {}. skipping fib update", rd);
1908                     macupdate = true;
1909                 }
1910             } else {
1911                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1912                 return;
1913             }
1914         }
1915         if (macupdate) {
1916             LOG.info("Removing Mac Fib entry rd {} mac{} nexthop {} ", rd, macaddress, nextHop);
1917             fibDSWriter.removeMacEntryFromDS(rd, macaddress);
1918             LOG.info("Removed Mac Fib entry rd {} prefix {} nexthop {} ", rd, macaddress, nextHop);
1919         } else {
1920             LOG.info("REMOVE: Removing Fib entry rd {} prefix {}", rd, prefix);
1921             fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nextHop);
1922             LOG.info("REMOVE: Removed Fib entry rd {} prefix {}", rd, prefix);
1923         }
1924     }
1925
1926     //TODO: below function is for testing purpose with cli
1927     public void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
1928         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
1929         fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nexthop);
1930         String vpnName = bgpUtil.getVpnNameFromRd(rd);
1931         if (vpnName != null) {
1932             vpnLinkService.leakRouteIfNeeded(vpnName, prefix, null /*nextHopList*/, 0 /*INVALID_LABEL*/,
1933                                              RouteOrigin.BGP, NwConstants.DEL_FLOW);
1934         }
1935     }
1936
1937     private static boolean isRouteModified(int label, Long labelInStaleMap) {
1938         return labelInStaleMap != null && !labelInStaleMap.equals(Long.valueOf(label));
1939     }
1940
1941     static class ReplayNbr {
1942         Neighbors nbr;
1943         boolean shouldRetry = false;
1944
1945         public Neighbors getNbr() {
1946             return nbr;
1947         }
1948
1949         public boolean isShouldRetry() {
1950             return shouldRetry;
1951         }
1952
1953         public void setShouldRetry(boolean retryNbr) {
1954             this.shouldRetry = retryNbr;
1955         }
1956
1957         ReplayNbr(Neighbors nbr, boolean shouldRetry) {
1958             this.nbr = nbr;
1959             this.shouldRetry = shouldRetry;
1960         }
1961     }
1962
1963     private static boolean replayNbrConfig(List<Neighbors> neighbors, BgpRouter br) {
1964         if (neighbors == null || neighbors.isEmpty()) {
1965             LOG.error("Replaying nbr configuration, received NULL list ");
1966             return true;
1967         }
1968
1969         List<ReplayNbr> replayNbrList = new ArrayList<>();
1970         for (Neighbors nbr : neighbors) {
1971             if (nbr != null) {
1972                 replayNbrList.add(new ReplayNbr(nbr, true));
1973             }
1974         }
1975         final int numberOfNbrRetries = 3;
1976         RetryOnException nbrRetry = new RetryOnException(numberOfNbrRetries);
1977         do {
1978             for (ReplayNbr replayNbr : replayNbrList) {
1979                 if (!replayNbr.isShouldRetry()) {
1980                     continue;
1981                 }
1982                 boolean replayDone = false;
1983                 LOG.debug("Replaying addNbr {}", replayNbr.getNbr().getAddress().getValue());
1984                 replayDone = false;
1985                 try {
1986                     final String md5password = extractMd5Secret(replayNbr.getNbr());
1987                     br.addNeighbor(replayNbr.getNbr().getAddress().getValue(),
1988                             replayNbr.getNbr().getRemoteAs().longValue(), md5password);
1989                     UpdateSource us = replayNbr.getNbr().getUpdateSource();
1990                     if (us != null) {
1991                         LOG.debug("Replaying updatesource along with nbr: {} US-ip: {} to peer {}",
1992                                 replayNbr.getNbr().getAddress().getValue(),
1993                                 us.getSourceIp().getValue(),
1994                                 us.getPeerIp().getValue());
1995                         br.addUpdateSource(us.getPeerIp().getValue(),
1996                                 us.getSourceIp().getValue());
1997                     }
1998                     replayDone = true;
1999                 } catch (TException | BgpRouterException eNbr) {
2000                     LOG.debug("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), eNbr);
2001                 }
2002                 boolean replaySuccess = true;
2003                 replaySuccess = replaySuccess && replayDone;
2004                 LOG.debug("Replay addNbr {} successful", replayNbr.getNbr().getAddress().getValue());
2005
2006                 //Update Source handling
2007                 UpdateSource us = replayNbr.getNbr().getUpdateSource();
2008                 if (replayDone == false && us != null) {
2009                     LOG.debug("Replaying updatesource {} to peer {}", us.getSourceIp().getValue(),
2010                             us.getPeerIp().getValue());
2011                     replayDone = false;
2012                     try {
2013                         br.addUpdateSource(us.getPeerIp().getValue(),
2014                                 us.getSourceIp().getValue());
2015                         replayDone = true;
2016                     } catch (TException | BgpRouterException eUs) {
2017                         LOG.debug("Replaying UpdateSource for Nbr {}, exception:",
2018                                 replayNbr.getNbr().getAddress().getValue(), eUs);
2019                     }
2020                     LOG.debug("Replay updatesource {} successful", us.getSourceIp().getValue());
2021                     replaySuccess = replaySuccess && replayDone;
2022                 }
2023                 //Ebgp Multihope
2024                 EbgpMultihop en = replayNbr.getNbr().getEbgpMultihop();
2025                 if (en != null) {
2026                     replayDone = false;
2027                     try {
2028                         br.addEbgpMultihop(en.getPeerIp().getValue(),
2029                                 en.getNhops().intValue());
2030                         replayDone = true;
2031                     } catch (TException | BgpRouterException eEbgpMhop) {
2032                         LOG.debug("Replaying EbgpMultihop for Nbr {}, exception: ",
2033                                 replayNbr.getNbr().getAddress().getValue(), eEbgpMhop);
2034                     }
2035                     replaySuccess = replaySuccess && replayDone;
2036                 }
2037
2038                 //afs
2039                 List<AddressFamilies> afs = replayNbr.getNbr().getAddressFamilies();
2040                 if (afs != null) {
2041                     for (AddressFamilies af : afs) {
2042                         af_afi afi = af_afi.findByValue(af.getAfi().intValue());
2043                         af_safi safi = af_safi.findByValue(af.getSafi().intValue());
2044                         replayDone = false;
2045                         try {
2046                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
2047                             replayDone = true;
2048                         } catch (TException | BgpRouterException eAFs) {
2049                             LOG.debug("Replaying AddressFamily for Nbr {}, exception:",
2050                                     replayNbr.getNbr().getAddress().getValue(), eAFs);
2051                         }
2052                         replaySuccess = replaySuccess && replayDone;
2053                     }
2054                 }
2055                 //replay is success --> no need to replay this nbr in next iteration.
2056                 replayNbr.setShouldRetry(replaySuccess ? false : true);
2057             }
2058         } while (nbrRetry.decrementAndRetry());
2059         boolean replaySuccess = true;
2060         for (ReplayNbr replayNbr : replayNbrList) {
2061             replaySuccess = replaySuccess && !replayNbr.isShouldRetry();
2062         }
2063         return replaySuccess;
2064     }
2065
2066     public String getConfigHost() {
2067         if (config == null) {
2068             return hostStartup;
2069         }
2070         ConfigServer ts = config.getConfigServer();
2071         return ts == null ? hostStartup : ts.getHost().getValue();
2072     }
2073
2074     public int getConfigPort() {
2075         if (config == null) {
2076             return Integer.parseInt(portStartup);
2077         }
2078         ConfigServer ts = config.getConfigServer();
2079         return ts == null ? Integer.parseInt(portStartup) :
2080                 ts.getPort().intValue();
2081     }
2082
2083     public Bgp getConfig() {
2084         AtomicInteger bgpDSretryCount = new AtomicInteger(DS_RETRY_COUNT);
2085         while (0 != bgpDSretryCount.decrementAndGet()) {
2086             try {
2087                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
2088                         InstanceIdentifier.create(Bgp.class)).orNull();
2089             } catch (ReadFailedException e) {
2090                 //Config DS may not be up, so sleep for 1 second and retry
2091                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
2092                 try {
2093                     Thread.sleep(WAIT_TIME_BETWEEN_EACH_TRY_MILLIS);
2094                 } catch (InterruptedException timerEx) {
2095                     LOG.debug("WAIT_TIME_BETWEEN_EACH_TRY_MILLIS, Timer got interrupted while waiting for"
2096                             + "config DS availability", timerEx);
2097                 }
2098             }
2099         }
2100         LOG.error("failed to get bgp config");
2101         return null;
2102     }
2103
2104     @SuppressWarnings("checkstyle:IllegalCatch")
2105     public synchronized boolean replay() throws InterruptedException, TimeoutException, ExecutionException {
2106         boolean replaySucceded = true;
2107         String host = getConfigHost();
2108         int port = getConfigPort();
2109         LOG.error("connecting  to bgp host {} ", host);
2110         boolean res = bgpRouter.connect(host, port);
2111         if (!res) {
2112             LOG.error("Cannot connect to BGP config server at {}:{}{}", host, port,
2113                     config != null ? "; Configuration Replay aborted" : "");
2114             return replaySucceded;
2115         }
2116         config = getConfig();
2117         if (config == null) {
2118             LOG.error("bgp config is empty nothing to push to bgp");
2119             return replaySucceded;
2120         }
2121         BgpRouter br = bgpRouter;
2122         AsId asId = config.getAsId();
2123         if (asId == null) {
2124             LOG.error("bgp as-id is null");
2125             return replaySucceded;
2126         }
2127         long asNum = asId.getLocalAs();
2128         IpAddress routerId = asId.getRouterId();
2129         String rid = routerId == null ? "" : new String(routerId.getValue());
2130         int stalepathTime = (int) getStalePathtime(RESTART_DEFAULT_GR, config.getAsId());
2131         boolean announceFbit = true;
2132         boolean replayDone = false;
2133         final int numberOfStartBgpRetries = 3;
2134         RetryOnException startBgpRetry = new RetryOnException(numberOfStartBgpRetries);
2135         do {
2136             try {
2137                 LOG.debug("Replaying BGPConfig ");
2138                 br.startBgp(asNum, rid, stalepathTime, announceFbit);
2139                 LOG.debug("Replay BGPConfig successful");
2140                 replayDone = true;
2141                 break;
2142             } catch (BgpRouterException bre) {
2143                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
2144                     LOG.debug("Starting the routesync for exception", bre);
2145                     startBgpRetry.errorOccured();
2146                     if (!startBgpRetry.shouldRetry()) {
2147                         LOG.debug("starting route sync for BgpRouter exception");
2148                         doRouteSync();
2149                     }
2150                 } else {
2151                     LOG.error("Replay: startBgp() received exception error {} : ",
2152                             bre.getErrorCode(), bre);
2153                     startBgpRetry.errorOccured();
2154                 }
2155             } catch (TApplicationException tae) {
2156                 if (tae.getType() == BgpRouterException.BGP_ERR_ACTIVE) {
2157                     LOG.debug("Starting the routesync for exception", tae);
2158                     startBgpRetry.errorOccured();
2159                     if (!startBgpRetry.shouldRetry()) {
2160                         LOG.debug("starting route sync for Thrift BGP_ERR_ACTIVE exception");
2161                         doRouteSync();
2162                     }
2163                 } else if (tae.getType() == BgpRouterException.BGP_ERR_COMMON_FAILURE) {
2164                     LOG.debug("Starting the routesync for AS-ID started exception", tae);
2165                     startBgpRetry.errorOccured();
2166                     if (!startBgpRetry.shouldRetry()) {
2167                         LOG.debug("starting route sync for Thrift BGP_ERR_COMMON_FAILURE exception");
2168                         doRouteSync();
2169                     }
2170                 } else {
2171                     LOG.error("Replay: startBgp() received exception type {}: ",
2172                             tae.getType(), tae);
2173                     startBgpRetry.errorOccured();
2174                 }
2175             } catch (Exception e) {
2176                 //not unusual. We may have restarted & BGP is already on
2177                 LOG.error("Replay:startBgp() received exception: ", e);
2178                 startBgpRetry.errorOccured();
2179             }
2180         } while (startBgpRetry.shouldRetry());
2181
2182         replaySucceded = replayDone;
2183
2184         startBgpCountersTask();
2185         startBgpAlarmsTask();
2186
2187         /*
2188          * commenting this due to a bug with QBGP. Will uncomment once QBGP fix is done.
2189          * This wont have any functional impacts
2190          */
2191         try {
2192             br.delayEOR(delayEorSeconds);
2193         } catch (TException | BgpRouterException e) {
2194             LOG.error("Replay: delayEOR() number of seconds to wait for EOR from ODL:", e);
2195         }
2196
2197         List<Neighbors> neighbors = config.getNeighbors();
2198         if (neighbors != null) {
2199             LOG.error("configuring existing Neighbors present for replay total neighbors {}", neighbors.size());
2200             boolean neighborConfigReplayResult = replayNbrConfig(neighbors, br);
2201             if (neighborConfigReplayResult == false) {
2202                 replaySucceded = false;
2203             }
2204         } else {
2205             LOG.error("no Neighbors present for replay config ");
2206         }
2207
2208         Logging logging = config.getLogging();
2209         if (logging != null) {
2210             try {
2211                 br.setLogging(logging.getFile(), logging.getLevel());
2212             } catch (TException | BgpRouterException e) {
2213                 LOG.error("Replay:setLogging() received exception", e);
2214             }
2215         }
2216
2217         GracefulRestart gracefulRestart = config.getGracefulRestart();
2218         if (gracefulRestart != null) {
2219             try {
2220                 br.addGracefulRestart(gracefulRestart.getStalepathTime().intValue());
2221             } catch (TException | BgpRouterException e) {
2222                 LOG.error("Replay:addGr() received exception", e);
2223             }
2224         }
2225
2226         List<Vrfs> vrfs = config.getVrfs();
2227         if (vrfs == null) {
2228             vrfs = new ArrayList<>();
2229         }
2230         for (Vrfs vrf : vrfs) {
2231             for (AddressFamiliesVrf adf : vrf.getAddressFamiliesVrf()) {
2232                 try {
2233                     br.addVrf(BgpUtil.getLayerType(adf), vrf.getRd(), vrf.getImportRts(),
2234                             vrf.getExportRts());
2235                 } catch (TException | BgpRouterException e) {
2236                     LOG.error("Replay:addVrf() received exception", e);
2237                 }
2238             }
2239         }
2240
2241         List<Networks> ln = config.getNetworks();
2242         if (ln != null) {
2243             for (Networks net : ln) {
2244                 String rd = net.getRd();
2245                 String pfxlen = net.getPrefixLen();
2246                 String nh = net.getNexthop().getValue();
2247                 Long label = net.getLabel();
2248                 int lbl = label == null ? 0 : label.intValue();
2249                 int l3vni = net.getL3vni() == null ? 0 : net.getL3vni().intValue();
2250                 int l2vni = net.getL2vni() == null ? 0 : net.getL2vni().intValue();
2251                 if (rd == null && lbl > 0) {
2252                     //LU prefix is being deleted.
2253                     rd = Integer.toString(lbl);
2254                 }
2255
2256                 BgpControlPlaneType protocolType = net.getBgpControlPlaneType();
2257                 int ethernetTag = net.getEthtag().intValue();
2258                 String esi = net.getEsi();
2259                 String macaddress = net.getMacaddress();
2260                 EncapType encapType = net.getEncapType();
2261                 String routerMac = net.getRoutermac();
2262
2263                 try {
2264                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni,
2265                             BgpUtil.convertToThriftProtocolType(protocolType),
2266                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
2267                 } catch (TException | BgpRouterException e) {
2268                     LOG.error("Replay:addPfx() received exception", e);
2269                 }
2270             }
2271         }
2272
2273
2274         List<Multipath> multipaths = config.getMultipath();
2275
2276         if (multipaths != null) {
2277             for (Multipath multipath : multipaths) {
2278                 if (multipath != null) {
2279                     af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
2280                     af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
2281
2282                     try {
2283                         if (multipath.isMultipathEnabled()) {
2284                             br.enableMultipath(afi, safi);
2285                         } else {
2286                             br.disableMultipath(afi, safi);
2287                         }
2288                     } catch (TException | BgpRouterException e) {
2289                         LOG.info("Replay:multipaths() received exception", e);
2290                     }
2291                 }
2292             }
2293         }
2294         List<VrfMaxpath> vrfMaxpaths = config.getVrfMaxpath();
2295         if (vrfMaxpaths != null) {
2296             for (VrfMaxpath vrfMaxpath : vrfMaxpaths) {
2297                 try {
2298                     br.multipaths(vrfMaxpath.getRd(), vrfMaxpath.getMaxpaths());
2299                 } catch (TException | BgpRouterException e) {
2300                     LOG.info("Replay:vrfMaxPath() received exception", e);
2301                 }
2302             }
2303         }
2304
2305         //send End of Rib Marker to Qthriftd.
2306         final int numberOfEORRetries = 3;
2307         replayDone = false;
2308         RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
2309         do {
2310             try {
2311                 br.sendEOR();
2312                 LOG.debug("Replay sendEOR {} successful");
2313                 replayDone = true;
2314                 break;
2315             } catch (Exception e) {
2316                 eorRetry.errorOccured();
2317                 LOG.error("Replay:sedEOR() received exception:", e);
2318             }
2319         } while (eorRetry.shouldRetry());
2320
2321         return replaySucceded && replayDone;
2322     }
2323
2324     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
2325         bgpUtil.update(iid, dto);
2326     }
2327
2328     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
2329         bgpUtil.delete(iid);
2330     }
2331
2332     public void startConfig(String bgpHost, int thriftPort) {
2333         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2334                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2335         InstanceIdentifier<ConfigServer> iid = iib.build();
2336         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
2337         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
2338                 .setPort((long) thriftPort).build();
2339         update(iid, dto);
2340     }
2341
2342     public void startBgp(long as, String routerId, int spt, boolean fbit) {
2343         IpAddress rid = routerId == null ? null : new IpAddress(routerId.toCharArray());
2344         Long staleTime = (long) spt;
2345         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2346                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2347         InstanceIdentifier<AsId> iid = iib.build();
2348         AsId dto = new AsIdBuilder().setLocalAs(as)
2349                 .setRouterId(rid)
2350                 .setStalepathTime(staleTime)
2351                 .setAnnounceFbit(fbit).build();
2352         update(iid, dto);
2353     }
2354
2355     public void addLogging(String fileName, String logLevel) {
2356         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2357                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2358         InstanceIdentifier<Logging> iid = iib.build();
2359         Logging dto = new LoggingBuilder().setFile(fileName)
2360                 .setLevel(logLevel).build();
2361         update(iid, dto);
2362     }
2363
2364     public void addGracefulRestart(int staleTime) {
2365         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2366                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
2367         InstanceIdentifier<GracefulRestart> iid = iib.build();
2368         GracefulRestart dto = new GracefulRestartBuilder()
2369                 .setStalepathTime((long) staleTime).build();
2370         update(iid, dto);
2371     }
2372
2373     public void addNeighbor(
2374             String nbrIp, long remoteAs, @Nullable final TcpMd5SignaturePasswordType md5Secret) {
2375         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2376         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2377                 InstanceIdentifier.builder(Bgp.class)
2378                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2379         InstanceIdentifier<Neighbors> iid = iib.build();
2380         TcpSecurityOption tcpSecOption = null;
2381         if (md5Secret != null) {
2382             tcpSecOption = new TcpMd5SignatureOptionBuilder().setTcpMd5SignaturePassword(md5Secret).build();
2383         } // else let tcpSecOption be null
2384         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
2385                 .setRemoteAs(remoteAs).setTcpSecurityOption(tcpSecOption).build();
2386         update(iid, dto);
2387     } // public addNeighbor(nbrIp, remoteAs, md5Secret)
2388
2389     public void addUpdateSource(String nbrIp, String srcIp) {
2390         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2391         Ipv4Address srcAddr = new Ipv4Address(srcIp);
2392         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2393                 InstanceIdentifier.builder(Bgp.class)
2394                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2395                         .child(UpdateSource.class);
2396         InstanceIdentifier<UpdateSource> iid = iib.build();
2397         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
2398                 .setSourceIp(srcAddr).build();
2399         update(iid, dto);
2400     }
2401
2402     public void addEbgpMultihop(String nbrIp, int hops) {
2403         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2404         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2405                 InstanceIdentifier.builder(Bgp.class)
2406                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2407                         .child(EbgpMultihop.class);
2408         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2409         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
2410                 .setNhops((long) hops).build();
2411         update(iid, dto);
2412     }
2413
2414     public void addAddressFamily(String nbrIp, int afi, int safi) {
2415         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2416         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2417                 InstanceIdentifier.builder(Bgp.class)
2418                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2419                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2420         InstanceIdentifier<AddressFamilies> iid = iib.build();
2421         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
2422                 .setAfi((long) afi).setSafi((long) safi).build();
2423         update(iid, dto);
2424     }
2425
2426     public void addPrefix(String rd, String macAddress, String pfx, List<String> nhList,
2427               VrfEntry.EncapType encapType, long lbl, long l3vni, long l2vni, String gatewayMac) {
2428         for (String nh : nhList) {
2429             Ipv4Address nexthop = nh != null ? new Ipv4Address(nh) : null;
2430             Long label = lbl;
2431             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
2432                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
2433             NetworksBuilder networksBuilder = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
2434                                                 .setLabel(label).setEthtag(BgpConstants.DEFAULT_ETH_TAG);
2435             buildVpnEncapSpecificInfo(networksBuilder, encapType, label, l3vni, l2vni, macAddress, gatewayMac);
2436             update(iid, networksBuilder.build());
2437         }
2438     }
2439
2440     private static void buildVpnEncapSpecificInfo(NetworksBuilder builder, VrfEntry.EncapType encapType, long label,
2441                                                   long l3vni, long l2vni, String macAddress, String gatewayMac) {
2442         if (encapType.equals(VrfEntry.EncapType.Mplsgre)) {
2443             builder.setLabel(label).setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLL3VPN)
2444                     .setEncapType(EncapType.GRE);
2445         } else {
2446             builder.setL3vni(l3vni).setL2vni(l2vni).setMacaddress(macAddress).setRoutermac(gatewayMac)
2447                     .setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLEVPN).setEncapType(EncapType.VXLAN);
2448         }
2449     }
2450
2451     // TODO: add LayerType as arg - supports command
2452     public void addVrf(String rd, List<String> irts, List<String> erts, AddressFamily addressFamily) {
2453         Vrfs vrf = bgpUtil.getVrfFromRd(rd);
2454         List<AddressFamiliesVrf> adfList = new ArrayList<>(1);
2455         if (vrf != null) {
2456             adfList = vrf.getAddressFamiliesVrf();
2457         }
2458         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2459         if (addressFamily.equals(AddressFamily.IPV4)) {
2460             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2461             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2462         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2463             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2464             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2465         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2466             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2467             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2468         }
2469         AddressFamiliesVrf adf = adfBuilder.build();
2470         adfList.add(adf);
2471         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib = InstanceIdentifier.builder(Bgp.class)
2472                 .child(Vrfs.class, new VrfsKey(rd));
2473         InstanceIdentifier<Vrfs> iid = iib.build();
2474         Vrfs dto = new VrfsBuilder().setRd(rd).setImportRts(irts)
2475             .setExportRts(erts).setAddressFamiliesVrf(adfList).build();
2476
2477         List<AddressFamiliesVrf> listAdFamilies = mapNewAdFamily.get(rd);
2478         if (listAdFamilies != null) {
2479             listAdFamilies.add(adf);
2480         } else {
2481             mapNewAdFamily.put(rd, adfList);
2482         }
2483
2484         try {
2485             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
2486         } catch (TransactionCommitFailedException e) {
2487             LOG.error("Error adding VRF to datastore", e);
2488             throw new RuntimeException(e);
2489         }
2490     }
2491
2492     public void stopConfig() {
2493         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2494                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2495         InstanceIdentifier<ConfigServer> iid = iib.build();
2496         delete(iid);
2497     }
2498
2499     public void stopBgp() {
2500         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2501                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2502         InstanceIdentifier<AsId> iid = iib.build();
2503         delete(iid);
2504     }
2505
2506     public void delLogging() {
2507         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2508                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2509         InstanceIdentifier<Logging> iid = iib.build();
2510         delete(iid);
2511     }
2512
2513     public void delGracefulRestart() {
2514         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2515                 InstanceIdentifier.builder(Bgp.class)
2516                         .child(GracefulRestart.class);
2517         InstanceIdentifier<GracefulRestart> iid = iib.build();
2518         delete(iid);
2519     }
2520
2521     public void delNeighbor(String nbrIp) {
2522         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2523         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2524                 InstanceIdentifier.builder(Bgp.class)
2525                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2526         InstanceIdentifier<Neighbors> iid = iib.build();
2527         delete(iid);
2528     }
2529
2530     public void delUpdateSource(String nbrIp) {
2531         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2532         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2533                 InstanceIdentifier.builder(Bgp.class)
2534                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2535                         .child(UpdateSource.class);
2536         InstanceIdentifier<UpdateSource> iid = iib.build();
2537         delete(iid);
2538     }
2539
2540     public void delEbgpMultihop(String nbrIp) {
2541         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2542         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2543                 InstanceIdentifier.builder(Bgp.class)
2544                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2545                         .child(EbgpMultihop.class);
2546         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2547         delete(iid);
2548     }
2549
2550     public void delAddressFamily(String nbrIp, int afi, int safi) {
2551         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2552         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2553                 InstanceIdentifier.builder(Bgp.class)
2554                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2555                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2556         InstanceIdentifier<AddressFamilies> iid = iib.build();
2557         delete(iid);
2558     }
2559
2560     public void delPrefix(String rd, String pfx) {
2561         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
2562                 InstanceIdentifier.builder(Bgp.class)
2563                         .child(Networks.class, new NetworksKey(pfx, rd));
2564         InstanceIdentifier<Networks> iid = iib.build();
2565         delete(iid);
2566     }
2567
2568     public boolean delVrf(String rd, AddressFamily addressFamily) {
2569         if (addressFamily == null) {
2570             LOG.error("delVrf: vrf {}, addressFamily invalid", rd);
2571             return false;
2572         }
2573         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2574         if (addressFamily.equals(AddressFamily.IPV4)) {
2575             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2576             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2577         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2578             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2579             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2580         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2581             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2582             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2583         }
2584         Vrfs vrfOriginal = bgpUtil.getVrfFromRd(rd);
2585         if (vrfOriginal == null) {
2586             LOG.error("delVrf: no vrf with existing rd {}. step aborted", rd);
2587             return false;
2588         }
2589
2590         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2591                 InstanceIdentifier.builder(Bgp.class)
2592                         .child(Vrfs.class, new VrfsKey(rd));
2593
2594         InstanceIdentifier<Vrfs> iid = iib.build();
2595
2596         @SuppressWarnings("static-access")
2597         InstanceIdentifier<Bgp> iid6 =  iid.builder(Bgp.class).build()
2598                 .child(Multipath.class, new MultipathKey(adfBuilder.getAfi(), adfBuilder.getSafi())).create(Bgp.class);
2599         InstanceIdentifierBuilder<Vrfs> iib3 = iid6.child(Vrfs.class, new VrfsKey(rd)).builder();
2600         InstanceIdentifier<Vrfs> iidFinal = iib3.build();
2601
2602         //** update or delete the vrfs with the rest of AddressFamilies already present in the last list
2603         AddressFamiliesVrf adfToDel = adfBuilder.build();
2604         List<AddressFamiliesVrf> adfListOriginal = vrfOriginal.getAddressFamiliesVrf() == null
2605                 ? new ArrayList<>() : vrfOriginal.getAddressFamiliesVrf();
2606         List<AddressFamiliesVrf> adfListToRemoveFromOriginal = new ArrayList<>();
2607         adfListOriginal.forEach(adf -> {
2608             if (adf.equals(adfToDel)) {
2609                 adfListToRemoveFromOriginal.add(adfToDel);
2610                 return;
2611             }
2612         });
2613         for (AddressFamiliesVrf adfToRemove : adfListToRemoveFromOriginal) {
2614             adfListOriginal.remove(adfToRemove);
2615             try {
2616                 SingleTransactionDataBroker.syncWrite(dataBroker,
2617                         LogicalDatastoreType.CONFIGURATION, iid, vrfOriginal);
2618             } catch (TransactionCommitFailedException e) {
2619                 LOG.error("delVrf: Error updating VRF to datastore", e);
2620                 throw new RuntimeException(e);
2621             }
2622         }
2623         if (adfListOriginal.isEmpty()) {
2624             delete(iidFinal);
2625             return true;
2626         }
2627         // not all is removed
2628         return false;
2629     }
2630
2631     public void setMultipathStatus(af_afi afi, af_safi safi, boolean enable) {
2632         long lafi = afi.getValue();
2633         long lsafi = safi.getValue();
2634
2635         InstanceIdentifier.InstanceIdentifierBuilder<Multipath> iib =
2636                 InstanceIdentifier
2637                         .builder(Bgp.class)
2638                         .child(Multipath.class,
2639                                 new MultipathKey(Long.valueOf(afi.getValue()), Long.valueOf(safi.getValue())));
2640
2641         Multipath dto = new MultipathBuilder().setAfi(lafi).setSafi(lsafi).setMultipathEnabled(enable).build();
2642         update(iib.build(), dto);
2643     }
2644
2645     public void multipaths(String rd, int maxpath) {
2646         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
2647                 InstanceIdentifier
2648                         .builder(Bgp.class)
2649                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
2650
2651         VrfMaxpath dto = new VrfMaxpathBuilder().setRd(rd).setMaxpaths(maxpath).build();
2652         update(iib.build(), dto);
2653     }
2654
2655     /*
2656     * Remove Stale Marked Routes after timer expiry.
2657     */
2658     private class RouteCleanup implements Callable<Integer> {
2659
2660         @Override
2661         public Integer call() {
2662             totalCleared = 0;
2663             try {
2664                 if (staledFibEntriesMap.isEmpty()) {
2665                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
2666                 } else {
2667                     for (String rd : staledFibEntriesMap.keySet()) {
2668                         if (Thread.interrupted()) {
2669                             return 0;
2670                         }
2671                         Map<String, Long> map = staledFibEntriesMap.get(rd);
2672                         if (map != null) {
2673                             for (String key : map.keySet()) {
2674                                 if (Thread.interrupted()) {
2675                                     return 0;
2676                                 }
2677                                 String prefix = extractPrefix(key);
2678                                 String nextHop = extractNextHop(key);
2679                                 totalCleared++;
2680                                 LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}, nextHop:{}",
2681                                         rd, prefix, nextHop);
2682                                 fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix, nextHop);
2683                             }
2684                         }
2685                     }
2686                 }
2687             } finally {
2688                 staledFibEntriesMap.clear();
2689             }
2690             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
2691             return 0;
2692         }
2693     }
2694
2695     /*
2696      * BGP restart scenario, ODL-BGP manager was/is running.
2697      * On re-sync notification, Get a copy of FIB database.
2698      */
2699     public void createStaleFibMap() {
2700         totalStaledCount = 0;
2701         try {
2702             staledFibEntriesMap.clear();
2703             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2704
2705             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2706                     LogicalDatastoreType.CONFIGURATION, id);
2707             if (fibEntries.isPresent()) {
2708                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2709                 for (VrfTables vrfTable : staleVrfTables) {
2710                     Map<String, Long> staleFibEntMap = new HashMap<>();
2711                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2712                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2713                             //Stale marking and cleanup is only meant for the routes learned through BGP.
2714                             continue;
2715                         }
2716                         if (Thread.interrupted()) {
2717                             break;
2718                         }
2719                         totalStaledCount++;
2720                         //Create MAP from staleVrfTables.
2721                         vrfEntry.getRoutePaths()
2722                                 .forEach(
2723                                     routePath -> staleFibEntMap.put(
2724                                             appendNextHopToPrefix(vrfEntry.getDestPrefix(),
2725                                                     routePath.getNexthopAddress()), routePath.getLabel()));
2726                     }
2727                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), staleFibEntMap);
2728                 }
2729             } else {
2730                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
2731             }
2732         } catch (ReadFailedException e) {
2733             LOG.error("createStaleFibMap:: error ", e);
2734         }
2735         LOG.error("created {} staled entries ", totalStaledCount);
2736     }
2737
2738     /*
2739      * BGP config remove scenario, Need to remove all the
2740      * external routes from FIB.
2741      */
2742     public void deleteExternalFibRoutes() {
2743         totalExternalRoutes = 0;
2744         totalExternalMacRoutes = 0;
2745         try {
2746             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2747
2748             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2749                     LogicalDatastoreType.CONFIGURATION, id);
2750             if (fibEntries.isPresent()) {
2751                 if (fibEntries.get().getVrfTables() == null) {
2752                     LOG.error("deleteExternalFibRoutes::getVrfTables is null");
2753                     return;
2754                 }
2755                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2756                 for (VrfTables vrfTable : staleVrfTables) {
2757                     String rd = vrfTable.getRouteDistinguisher();
2758                     if (vrfTable.getVrfEntry() != null) {
2759                         for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2760                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2761                                 //route cleanup is only meant for the routes learned through BGP.
2762                                 continue;
2763                             }
2764                             totalExternalRoutes++;
2765                             fibDSWriter.removeFibEntryFromDS(rd, vrfEntry.getDestPrefix());
2766                         }
2767                     } else if (vrfTable.getMacVrfEntry() != null) {
2768                         for (MacVrfEntry macEntry : vrfTable.getMacVrfEntry()) {
2769                             if (RouteOrigin.value(macEntry.getOrigin()) != RouteOrigin.BGP) {
2770                                 //route cleanup is only meant for the routes learned through BGP.
2771                                 continue;
2772                             }
2773                             totalExternalMacRoutes++;
2774                             fibDSWriter.removeMacEntryFromDS(rd, macEntry.getMac());
2775                         }
2776                     }
2777                 }
2778             } else {
2779                 LOG.error("deleteExternalFibRoutes:: FIBentries.class is not present");
2780             }
2781         } catch (ReadFailedException e) {
2782             LOG.error("deleteExternalFibRoutes:: error ", e);
2783         }
2784         LOG.debug("deleted {} fib entries {} mac entries", totalExternalRoutes, totalExternalMacRoutes);
2785     }
2786
2787     public boolean addToRt2TepMap(String rd, String tepIp, String mac, Long l2vni) {
2788         boolean isFirstMacUpdateFromTep = false;
2789         if (rt2TepMap.containsKey(rd)) {
2790             if (rt2TepMap.get(rd).containsKey(tepIp)) {
2791                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and tep-ip {}. No Elan DS write required",
2792                         mac, l2vni, rd, tepIp);
2793                 rt2TepMap.get(rd).get(tepIp).put(mac, l2vni);
2794             } else {
2795                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and new tep-ip {}",
2796                         mac, l2vni, rd, tepIp);
2797                 isFirstMacUpdateFromTep = true;
2798                 Map<String, Long> macList = new HashMap<>();
2799                 macList.put(mac, l2vni);
2800                 rt2TepMap.get(rd).put(tepIp, macList);
2801             }
2802         } else {
2803             LOG.debug("RT2 with mac {} l2vni {} from new rd {} and tep ip {}",
2804                     mac, l2vni, rd, tepIp);
2805             isFirstMacUpdateFromTep = true;
2806             Map<String, Long> macList = new HashMap<>();
2807             macList.put(mac, l2vni);
2808             Map<String, Map<String, Long>> tepIpMacMap = new HashMap<>();
2809             tepIpMacMap.put(tepIp, macList);
2810             rt2TepMap.put(rd, tepIpMacMap);
2811         }
2812         return isFirstMacUpdateFromTep;
2813     }
2814
2815     public boolean deleteFromRt2TepMap(String rd, String tepIp, String mac) {
2816         boolean isLastMacUpdateFromTep = false;
2817         LOG.debug("RT2 withdraw with rd {} mac {} tep-ip {} ", rd, mac, tepIp);
2818         if (rt2TepMap.containsKey(rd)) {
2819             if (rt2TepMap.get(rd).containsKey(tepIp)) {
2820                 if (rt2TepMap.get(rd).get(tepIp).containsKey(mac)) {
2821                     LOG.debug("RT2 Withdraw : Removing the mac {} from Map", mac);
2822                     rt2TepMap.get(rd).get(tepIp).remove(mac);
2823                     if (rt2TepMap.get(rd).get(tepIp).isEmpty()) {
2824                         isLastMacUpdateFromTep = true;
2825                         LOG.debug("RT2 Withdraw : Removing the tep-ip {} from Map", tepIp);
2826                         rt2TepMap.get(rd).remove(tepIp);
2827                         if (rt2TepMap.get(rd).isEmpty()) {
2828                             LOG.debug("RT2 Withdraw : Removing the rd {} from Map", rd);
2829                             rt2TepMap.remove(rd);
2830                         }
2831                     }
2832                 }
2833             }
2834         }
2835         return isLastMacUpdateFromTep;
2836     }
2837
2838     public Collection<String> getTepIPs(String rd) {
2839         final Map<String, Map<String, Long>> tepIpMap = rt2TepMap.get(rd);
2840         return tepIpMap != null ? tepIpMap.keySet() : Collections.emptyList();
2841     }
2842
2843     public boolean isBgpConnected() {
2844         return bgpRouter.isBgpConnected();
2845     }
2846
2847     public long getLastConnectedTS() {
2848         return bgpRouter.getLastConnectedTS();
2849     }
2850
2851     public long getConnectTS() {
2852         return bgpRouter.getConnectTS();
2853     }
2854
2855     public long getStartTS() {
2856         return bgpRouter.getStartTS();
2857     }
2858
2859     public TTransport getTransport() {
2860         return bgpRouter.getTransport();
2861     }
2862
2863     public int getTotalStaledCount() {
2864         return totalStaledCount;
2865     }
2866
2867     public int getTotalCleared() {
2868         return totalCleared;
2869     }
2870
2871     public BgpCounters getBgpCounters() {
2872         return bgpCountersReference.get();
2873     }
2874
2875     private void startBgpCountersTask() {
2876         if (getBgpCounters() == null && bgpCountersReference.compareAndSet(null,
2877                 new BgpCounters(getBgpSdncMipIp(), metricProvider))) {
2878             bgpCountersTask = executor.scheduleAtFixedRate(bgpCountersReference.get(), 0, 120 * 1000,
2879                     TimeUnit.MILLISECONDS);
2880             LOG.info("Bgp Counters task scheduled for every two minutes.");
2881
2882             bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
2883         }
2884     }
2885
2886     private void stopBgpCountersTask() {
2887         final BgpCounters bgpCounters = bgpCountersReference.getAndSet(null);
2888         if (bgpCounters != null) {
2889             bgpCountersTask.cancel(true);
2890             bgpCounters.close();
2891         }
2892     }
2893
2894     private void startBgpAlarmsTask() {
2895         if (getBgpAlarms() == null && bgpAlarmsReference.compareAndSet(null, new BgpAlarms(this))) {
2896             bgpAlarmsReference.get().init();
2897             bgpAlarmsTask = executor.scheduleAtFixedRate(bgpAlarmsReference.get(), 0, 60 * 1000, TimeUnit.MILLISECONDS);
2898             LOG.info("Bgp Alarms task scheduled for every minute.");
2899         }
2900     }
2901
2902     private void stopBgpAlarmsTask() {
2903         final BgpAlarms bgpAlarms = bgpAlarmsReference.getAndSet(null);
2904         if (bgpAlarms != null) {
2905             bgpAlarmsTask.cancel(true);
2906             bgpAlarms.close();
2907         }
2908     }
2909
2910     public BgpAlarms getBgpAlarms() {
2911         return bgpAlarmsReference.get();
2912     }
2913
2914     private static String appendNextHopToPrefix(String prefix, String nextHop) {
2915         return prefix + ":" + nextHop;
2916     }
2917
2918     private static String extractPrefix(String prefixNextHop) {
2919         return prefixNextHop.split(":")[0];
2920     }
2921
2922     private static String extractNextHop(String prefixNextHop) {
2923         return prefixNextHop.split(":")[1];
2924     }
2925
2926     private static String extractMd5Secret(final Neighbors val) {
2927         String md5Secret = null;
2928         TcpSecurityOption tcpSecOpt = val.getTcpSecurityOption();
2929         if (tcpSecOpt != null) {
2930             if (tcpSecOpt instanceof TcpMd5SignatureOption) {
2931                 md5Secret = ((TcpMd5SignatureOption) tcpSecOpt).getTcpMd5SignaturePassword().getValue();
2932             } else { // unknown TcpSecurityOption
2933                 LOG.debug("neighbors  Ignored unknown tcp-security-option of peer {}", val.getAddress().getValue());
2934             }
2935         }
2936         return md5Secret;
2937     } // private method extractMd5Secret
2938 }