bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.net.InetAddresses;
  13 import com.google.common.util.concurrent.ThreadFactoryBuilder;
  14 import io.netty.util.concurrent.GlobalEventExecutor;
  15 import java.io.IOException;
  16 import java.lang.reflect.Constructor;
  17 import java.lang.reflect.InvocationTargetException;
  18 import java.net.InetAddress;
  19 import java.net.InetSocketAddress;
  20 import java.net.NetworkInterface;
  21 import java.net.SocketException;
  22 import java.util.ArrayList;
  23 import java.util.Collection;
  24 import java.util.Collections;
  25 import java.util.Enumeration;
  26 import java.util.HashMap;
  27 import java.util.Iterator;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Timer;
  32 import java.util.concurrent.Callable;
  33 import java.util.concurrent.ConcurrentHashMap;
  34 import java.util.concurrent.CountDownLatch;
  35 import java.util.concurrent.Executors;
  36 import java.util.concurrent.Future;
  37 import java.util.concurrent.ScheduledExecutorService;
  38 import java.util.concurrent.ScheduledFuture;
  39 import java.util.concurrent.TimeUnit;
  40 import java.util.concurrent.atomic.AtomicInteger;
  41 import java.util.concurrent.atomic.AtomicReference;
  42 import javax.annotation.Nullable;
  43 import javax.annotation.PreDestroy;
  44 import javax.inject.Inject;
  45 import javax.inject.Singleton;
  46 import org.apache.thrift.TApplicationException;
  47 import org.apache.thrift.TException;
  48 import org.apache.thrift.transport.TTransport;
  49 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  50 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  51 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  52 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  53 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  54 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  55 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  56 import org.opendaylight.genius.mdsalutil.NwConstants;
  57 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  58 import org.opendaylight.infrautils.metrics.MetricProvider;
  59 import org.opendaylight.mdsal.eos.binding.api.Entity;
  60 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipCandidateRegistration;
  61 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipListenerRegistration;
  62 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  63 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  64 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  65 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  66 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  67 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  68 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  69 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  70 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  71 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  72 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  73 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  74 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  75 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  76 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  77 import org.opendaylight.netvirt.bgpmanager.thrift.gen.protocol_type;
  78 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  79 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  80 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  81 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  82 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.AddressFamily;
  83 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  84 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
  85 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EncapType;
  86 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.TcpMd5SignaturePasswordType;
  87 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
  88 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
  90 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
  92 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
  94 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Multipath;
  96 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathKey;
  98 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Neighbors;
  99 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsKey;
 101 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Networks;
 102 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksKey;
 104 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpath;
 105 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathKey;
 107 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Vrfs;
 108 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsKey;
 110 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamilies;
 111 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesKey;
 113 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihop;
 114 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.EbgpMultihopBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSource;
 116 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.UpdateSourceBuilder;
 117 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfs.AddressFamiliesVrf;
 118 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfs.AddressFamiliesVrfBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.TcpSecurityOption;
 120 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOption;
 121 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOptionBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
 123 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.macvrfentries.MacVrfEntry;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 130 import org.opendaylight.yangtools.yang.binding.DataObject;
 131 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 132 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 133 import org.osgi.framework.BundleContext;
 134 import org.osgi.util.tracker.ServiceTracker;
 135 import org.slf4j.Logger;
 136 import org.slf4j.LoggerFactory;
 137
 138 @Singleton
 139 public class BgpConfigurationManager {
 140     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 141
 142     // to have stale FIB map (RD, Prefix)
 143     //  number of seconds wait for route sync-up between ODL and BGP
 144     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 145     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 146     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 147     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 148     private static final String DEF_UPORT = "6644";
 149     private static final String DEF_CHOST = "255.255.255.255"; // Invalid Host IP
 150     private static final String DEF_CPORT = "7644";
 151     private static final String DEF_BGP_SDNC_MIP = "127.0.0.1";
 152     //vpnservice.bgp.thrift.bgp.mip is the MIP present with ODL. Here we open 6644 port
 153     private static final String BGP_SDNC_MIP = "vpnservice.bgp.thrift.bgp.mip";
 154     private static final int RESTART_DEFAULT_GR = 90;
 155     private static final int DS_RETRY_COUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
 156     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
 157     private static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 158     private static final String BGP_EOR_DELAY = "vpnservice.bgp.eordelay";
 159     private static final String DEF_BGP_EOR_DELAY = "1800";
 160     private static final String BGP_ENTITY_NAME = "bgp";
 161     private static final String ADD_WARN = "Config store updated; undo with Delete if needed.";
 162     private static final String DEL_WARN = "Config store updated; undo with Add if needed.";
 163     private static final String UPD_WARN = "Update operation not supported; Config store updated;"
 164             + " restore with another Update if needed.";
 165
 166     private static final Class<?>[] REACTORS = {
 167         ConfigServerReactor.class, AsIdReactor.class,
 168         GracefulRestartReactor.class, LoggingReactor.class,
 169         NeighborsReactor.class, UpdateSourceReactor.class,
 170         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 171         NetworksReactor.class, VrfsReactor.class, BgpReactor.class,
 172         MultipathReactor.class, VrfMaxpathReactor.class
 173     };
 174
 175     private IBgpManager bgpManager;
 176     private final DataBroker dataBroker;
 177     private final FibDSWriter fibDSWriter;
 178     private final IVpnLinkService vpnLinkService;
 179     private final BundleContext bundleContext;
 180     private final BgpUtil bgpUtil;
 181     private volatile Bgp config;
 182     private final BgpRouter bgpRouter;
 183     private final BgpSyncHandle bgpSyncHandle = new BgpSyncHandle();
 184     private volatile BgpThriftService bgpThriftService = null;
 185     private final int delayEorSeconds;
 186
 187     private final CountDownLatch initer = new CountDownLatch(1);
 188
 189     private final String hostStartup;
 190     private final String portStartup;
 191
 192     private final AtomicReference<BgpCounters> bgpCountersReference = new AtomicReference<>();
 193     private ScheduledFuture<?> bgpCountersTask;
 194
 195     private final AtomicReference<BgpAlarms> bgpAlarmsReference = new AtomicReference<>();
 196     private ScheduledFuture<?> bgpAlarmsTask;
 197
 198     private Future<?> lastReplayJobFt;
 199     private ScheduledFuture<?> routeCleanupFuture;
 200
 201     private long staleStartTime;
 202     private long staleEndTime;
 203     private long cfgReplayStartTime;
 204     private long cfgReplayEndTime;
 205     private long staleCleanupTime;
 206     private int totalStaledCount;
 207     private int totalCleared;
 208     private int totalExternalRoutes;
 209     private int totalExternalMacRoutes;
 210
 211     private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(
 212             new ThreadFactoryBuilder().setNameFormat("BgpConfigurationManager-%d").setDaemon(true).build());
 213
 214     /**
 215      * this map store the new address families to send to quagga. When it is sended you must clear it.
 216      * The keys String are rd (route distinguisher).
 217      * */
 218     private final ConcurrentHashMap<String, List<AddressFamiliesVrf>> mapNewAdFamily = new ConcurrentHashMap<>();
 219
 220     // map<rd, map<prefix/len:nexthop, label>>
 221     private final Map<String, Map<String, Long>> staledFibEntriesMap = new ConcurrentHashMap<>();
 222
 223     // map<rd, map<tep-ip, map<mac, l2vni>>>
 224     private final Map<String, Map<String, Map<String, Long>>> rt2TepMap = new ConcurrentHashMap<>();
 225
 226     private final List<AutoCloseable> listeners = new ArrayList<>();
 227
 228     private final EntityOwnershipUtils entityOwnershipUtils;
 229     private final EntityOwnershipCandidateRegistration candidateRegistration;
 230     private final EntityOwnershipListenerRegistration entityListenerRegistration;
 231     private final MetricProvider metricProvider;
 232
 233     @Inject
 234     public BgpConfigurationManager(final DataBroker dataBroker,
 235             final EntityOwnershipService entityOwnershipService,
 236             final FibDSWriter fibDSWriter,
 237             final IVpnLinkService vpnLinkSrvce,
 238             final BundleContext bundleContext,
 239             final BgpUtil bgpUtil,
 240             final MetricProvider metricProvider) {
 241         this.dataBroker = dataBroker;
 242         this.fibDSWriter = fibDSWriter;
 243         this.vpnLinkService = vpnLinkSrvce;
 244         this.bundleContext = bundleContext;
 245         this.bgpUtil = bgpUtil;
 246         this.metricProvider = metricProvider;
 247         hostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 248         portStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 249         LOG.info("ConfigServer at {}:{}", hostStartup, portStartup);
 250         VtyshCli.setHostAddr(hostStartup);
 251         ClearBgpCli.setHostAddr(hostStartup);
 252         bgpRouter = BgpRouter.newInstance(this::getConfig, this::isBGPEntityOwner);
 253         delayEorSeconds = Integer.parseInt(getProperty(BGP_EOR_DELAY, DEF_BGP_EOR_DELAY));
 254
 255         entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 256
 257         candidateRegistration = registerEntityCandidate(entityOwnershipService);
 258         entityListenerRegistration = registerEntityListener(entityOwnershipService);
 259
 260         /*register callbacks for reactors, shall be called after EoS registration.
 261          * as listeners user EoS service to identify Owner node. Listener call-backs
 262          * can get triggered immediately after registeration (before EoS register complete)
 263         */
 264         registerCallbacks();
 265
 266         LOG.info("BGP Configuration manager initialized");
 267         initer.countDown();
 268
 269         GlobalEventExecutor.INSTANCE.execute(() -> {
 270             ServiceTracker<IBgpManager, ?> tracker = null;
 271             try {
 272                 tracker = new ServiceTracker<>(bundleContext, IBgpManager.class, null);
 273                 tracker.open();
 274                 bgpManager = (IBgpManager) tracker.waitForService(TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES));
 275                 Preconditions.checkState(bgpManager != null, "IBgpManager service not found");
 276             } catch (InterruptedException e) {
 277                 throw new IllegalStateException("Error retrieving IBgpManager service", e);
 278             } finally {
 279                 if (tracker != null) {
 280                     tracker.close();
 281                 }
 282             }
 283
 284             String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
 285             if (InetAddresses.isInetAddress(getBgpSdncMipIp())) {
 286                 InetSocketAddress bgpThriftServerSocketAddr = new InetSocketAddress(getBgpSdncMipIp(),
 287                         Integer.parseInt(updatePort));
 288                 bgpThriftService = new BgpThriftService(bgpThriftServerSocketAddr, bgpManager, this);
 289                 if (isBGPEntityOwner()) {
 290                     //I am EoS owner of BGP, opening bgp thrift UPDATE-SERVER port.
 291                     LOG.info("BGP Configuration manager initialized: UPDATE-SERVER started");
 292                     bgpThriftService.start();
 293                 }
 294                 LOG.info("UPDATE server started :ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 295             } else {
 296                 LOG.error("Failed to init UPDATE server invalid ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 297             }
 298             LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 299         });
 300     }
 301
 302     public String getBgpSdncMipIp() {
 303         return getProperty(BGP_SDNC_MIP, DEF_BGP_SDNC_MIP);
 304     }
 305
 306     public long getStaleCleanupTime() {
 307         return staleCleanupTime;
 308     }
 309
 310     public void setStaleCleanupTime(long staleCleanupTime) {
 311         this.staleCleanupTime = staleCleanupTime;
 312     }
 313
 314     public long getCfgReplayEndTime() {
 315         return cfgReplayEndTime;
 316     }
 317
 318     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 319         this.cfgReplayEndTime = cfgReplayEndTime;
 320     }
 321
 322     public long getCfgReplayStartTime() {
 323         return cfgReplayStartTime;
 324     }
 325
 326     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 327         this.cfgReplayStartTime = cfgReplayStartTime;
 328     }
 329
 330     public long getStaleEndTime() {
 331         return staleEndTime;
 332     }
 333
 334     public void setStaleEndTime(long staleEndTime) {
 335         this.staleEndTime = staleEndTime;
 336     }
 337
 338     public long getStaleStartTime() {
 339         return staleStartTime;
 340     }
 341
 342     public void setStaleStartTime(long staleStartTime) {
 343         this.staleStartTime = staleStartTime;
 344     }
 345
 346     private Object createListener(Class<?> cls) {
 347         try {
 348             Constructor<?> ctor = cls.getConstructor(BgpConfigurationManager.class);
 349             return ctor.newInstance(this);
 350         } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException
 351                 e) {
 352             LOG.error("Failed to create listener object", e);
 353             return null;
 354         }
 355     }
 356
 357     private void registerCallbacks() {
 358         for (Class<?> reactor : REACTORS) {
 359             Object obj = createListener(reactor);
 360             if (obj != null) {
 361                 AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 362                 dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 363                 listeners.add(dcl);
 364             }
 365         }
 366     }
 367
 368     @SuppressWarnings("checkstyle:IllegalCatch")
 369     @PreDestroy
 370     public void close() {
 371         executor.shutdown();
 372
 373         if (bgpThriftService != null) {
 374             bgpThriftService.stop();
 375             bgpThriftService = null;
 376         }
 377
 378         if (isBgpConnected()) {
 379             //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 380             bgpRouter.disconnect();
 381         }
 382
 383         if (candidateRegistration != null) {
 384             candidateRegistration.close();
 385         }
 386
 387         entityListenerRegistration.close();
 388
 389         listeners.forEach(l -> {
 390             try {
 391                 l.close();
 392             } catch (Exception e) {
 393                 LOG.warn("Error closing {}", l ,e);
 394             }
 395         });
 396
 397         LOG.info("{} close", getClass().getSimpleName());
 398     }
 399
 400     private String getProperty(String var, String def) {
 401         String property = bundleContext.getProperty(var);
 402         return property == null ? def : property;
 403     }
 404
 405     private EntityOwnershipCandidateRegistration registerEntityCandidate(
 406             final EntityOwnershipService entityOwnershipService) {
 407         try {
 408             return entityOwnershipService.registerCandidate(
 409                     new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME));
 410         } catch (CandidateAlreadyRegisteredException e) {
 411             LOG.error("failed to register bgp entity", e);
 412             return null;
 413         }
 414     }
 415
 416     private EntityOwnershipListenerRegistration registerEntityListener(
 417             final EntityOwnershipService entityOwnershipService) {
 418         return entityOwnershipService.registerListener(BGP_ENTITY_TYPE_FOR_OWNERSHIP, ownershipChange -> {
 419             LOG.trace("entity owner change event fired: {}", ownershipChange);
 420
 421             if (ownershipChange.getState() == EntityOwnershipChangeState.LOCAL_OWNERSHIP_GRANTED) {
 422                 LOG.trace("This PL is the Owner");
 423                 if (bgpThriftService != null) {
 424                     //opening UPDATE-SERVER port.
 425                     bgpThriftService.start();
 426                 } else {
 427                     LOG.error("I am the owner of BGP entity, but bgpThriftService is not initialized yet");
 428                 }
 429                 bgpRestarted();
 430             } else {
 431                 LOG.debug("Not owner: hasOwner: {}, isOwner: {}", ownershipChange.getState().hasOwner(),
 432                         ownershipChange.getState().isOwner());
 433                 if (bgpThriftService != null && bgpThriftService.isBgpThriftServiceStarted()) {
 434                     //close the bgp Thrift Update-SERVER port opened on non-Entity Owner
 435                     bgpThriftService.stop();
 436                     bgpThriftService = null;
 437                 }
 438                 if (isBgpConnected()) {
 439                     //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 440                     bgpRouter.disconnect();
 441                 }
 442             }
 443         });
 444     }
 445
 446     public boolean isBGPEntityOwner() {
 447         if (entityOwnershipUtils == null) {
 448             LOG.error("entityOwnershipUtils is NULL when listener callbacks fired");
 449             return false;
 450         }
 451         return entityOwnershipUtils.isEntityOwner(new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME), 0, 1);
 452     }
 453
 454     public Bgp get() {
 455         config = getConfig();
 456         return config;
 457     }
 458
 459     public class ConfigServerReactor
 460             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 461             implements ClusteredDataTreeChangeListener<ConfigServer> {
 462         private static final String YANG_OBJ = "config-server ";
 463
 464         public ConfigServerReactor() {
 465             super(ConfigServer.class, ConfigServerReactor.class);
 466         }
 467
 468         @Override
 469         protected void add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 470             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 471             if (!isBGPEntityOwner()) {
 472                 return;
 473             }
 474
 475             try {
 476                 initer.await();
 477             } catch (InterruptedException e) {
 478                 // Ignored
 479             }
 480             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 481             bgpRouter.configServerUpdated();
 482             synchronized (BgpConfigurationManager.this) {
 483                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 484                         val.getPort().intValue());
 485                 if (!res) {
 486                     LOG.error(YANG_OBJ + "Add failed; " + ADD_WARN);
 487                 }
 488             }
 489             VtyshCli.setHostAddr(val.getHost().getValue());
 490             ClearBgpCli.setHostAddr(val.getHost().getValue());
 491         }
 492
 493         @Override
 494         protected ConfigServerReactor getDataTreeChangeListener() {
 495             return ConfigServerReactor.this;
 496         }
 497
 498         @Override
 499         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 500             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 501         }
 502
 503         @Override
 504         protected void remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 505             LOG.trace("received bgp disconnect");
 506             if (!isBGPEntityOwner()) {
 507                 return;
 508             }
 509
 510             bgpRouter.configServerUpdated();
 511
 512             synchronized (BgpConfigurationManager.this) {
 513                 Bgp conf = getConfig();
 514                 if (conf != null) {
 515                     AsId asId = conf.getAsId();
 516                     if (asId != null) {
 517                         long asNum = asId.getLocalAs();
 518                         try {
 519                             bgpRouter.stopBgp(asNum);
 520                             stopBgpCountersTask();
 521                             stopBgpAlarmsTask();
 522                         } catch (TException | BgpRouterException e) {
 523                             LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 524                         }
 525                     } else {
 526                         LOG.debug("bgp as-id is null while removing config-server");
 527                     }
 528                 } else {
 529                     LOG.error("Config Null while removing the config-server");
 530                 }
 531                 bgpRouter.disconnect();
 532             }
 533         }
 534
 535         @Override
 536         protected void update(InstanceIdentifier<ConfigServer> iid,
 537                 ConfigServer oldval, ConfigServer newval) {
 538             LOG.trace("received bgp Connection update");
 539             if (!isBGPEntityOwner()) {
 540                 return;
 541             }
 542             LOG.error(YANG_OBJ + UPD_WARN);
 543         }
 544     }
 545
 546     private BgpRouter getClient(String yangObj) {
 547         if (bgpRouter == null || !bgpRouter.isBgpConnected()) {
 548             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 549             return null;
 550         }
 551         return bgpRouter;
 552     }
 553
 554     public class AsIdReactor
 555             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 556             implements ClusteredDataTreeChangeListener<AsId> {
 557
 558         private static final String YANG_OBJ = "as-id ";
 559
 560         public AsIdReactor() {
 561             super(AsId.class, AsIdReactor.class);
 562         }
 563
 564         @Override
 565         protected void add(InstanceIdentifier<AsId> iid, AsId val) {
 566             LOG.error("received bgp add asid {}", val);
 567             if (!isBGPEntityOwner()) {
 568                 return;
 569             }
 570             LOG.debug("received add router config asNum {}", val.getLocalAs());
 571             synchronized (BgpConfigurationManager.this) {
 572                 BgpRouter br = getClient(YANG_OBJ);
 573                 if (br == null) {
 574                     LOG.debug("{} Unable to process add for asNum {}; {} {}", YANG_OBJ, val.getLocalAs(),
 575                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 576                     return;
 577                 }
 578                 bgpRestarted();
 579
 580                 startBgpCountersTask();
 581                 startBgpAlarmsTask();
 582             }
 583         }
 584
 585         @Override
 586         protected AsIdReactor getDataTreeChangeListener() {
 587             return AsIdReactor.this;
 588         }
 589
 590         @Override
 591         protected InstanceIdentifier<AsId> getWildCardPath() {
 592             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 593         }
 594
 595         @Override
 596         protected void remove(InstanceIdentifier<AsId> iid, AsId val) {
 597             LOG.error("received delete router config asNum {}", val.getLocalAs());
 598             if (!isBGPEntityOwner()) {
 599                 return;
 600             }
 601             synchronized (BgpConfigurationManager.this) {
 602                 long asNum = val.getLocalAs();
 603                 BgpRouter br = getClient(YANG_OBJ);
 604                 if (br == null) {
 605                     LOG.debug("{} Unable to process remove for asNum {}; {} {}", YANG_OBJ, asNum,
 606                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 607                     return;
 608                 }
 609                 try {
 610                     br.stopBgp(asNum);
 611                 } catch (TException | BgpRouterException e) {
 612                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 613                 }
 614
 615                 stopBgpCountersTask();
 616                 stopBgpAlarmsTask();
 617
 618                 Bgp conf = getConfig();
 619                 if (conf == null) {
 620                     LOG.error("Config Null while removing the as-id");
 621                     return;
 622                 }
 623                 LOG.debug("Removing external routes from FIB");
 624                 deleteExternalFibRoutes();
 625                 List<Neighbors> nbrs = conf.getNeighbors();
 626                 if (nbrs != null && nbrs.size() > 0) {
 627                     LOG.error("Tring to remove the as-id when neighbor config is already present");
 628                     for (Neighbors nbr : nbrs) {
 629                         LOG.debug("Removing Neighbor {} from Data store", nbr.getAddress().getValue());
 630                         delNeighbor(nbr.getAddress().getValue());
 631                     }
 632                 }
 633             }
 634         }
 635
 636         @Override
 637         protected void update(InstanceIdentifier<AsId> iid,
 638                 AsId oldval, AsId newval) {
 639             if (!isBGPEntityOwner()) {
 640                 return;
 641             }
 642             LOG.error(YANG_OBJ + UPD_WARN);
 643         }
 644     }
 645
 646     public class GracefulRestartReactor
 647             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 648             implements ClusteredDataTreeChangeListener<GracefulRestart> {
 649
 650         private static final String YANG_OBJ = "graceful-restart ";
 651
 652         public GracefulRestartReactor() {
 653             super(GracefulRestart.class, GracefulRestartReactor.class);
 654         }
 655
 656         @Override
 657         protected void add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 658             if (!isBGPEntityOwner()) {
 659                 return;
 660             }
 661             synchronized (BgpConfigurationManager.this) {
 662                 int stalePathTime = val.getStalepathTime().intValue();
 663                 BgpRouter br = getClient(YANG_OBJ);
 664                 if (br == null) {
 665                     LOG.error("{} Unable to add stale-path time {}; {} {}", YANG_OBJ, stalePathTime,
 666                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 667                     return;
 668                 }
 669                 try {
 670                     br.addGracefulRestart(stalePathTime);
 671                 } catch (TException | BgpRouterException e) {
 672                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 673                 }
 674             }
 675         }
 676
 677         @Override
 678         protected GracefulRestartReactor getDataTreeChangeListener() {
 679             return GracefulRestartReactor.this;
 680         }
 681
 682         @Override
 683         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 684             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 685         }
 686
 687         @Override
 688         protected void remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 689             if (!isBGPEntityOwner()) {
 690                 return;
 691             }
 692             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 693             synchronized (BgpConfigurationManager.this) {
 694                 BgpRouter br = getClient(YANG_OBJ);
 695                 if (br == null) {
 696                     LOG.error("{} Unable to delete stale-path time; {} {}", YANG_OBJ,
 697                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 698                     return;
 699                 }
 700                 try {
 701                     br.delGracefulRestart();
 702                 } catch (TException | BgpRouterException e) {
 703                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 704                 }
 705             }
 706         }
 707
 708         @Override
 709         protected void update(InstanceIdentifier<GracefulRestart> iid,
 710                 GracefulRestart oldval, GracefulRestart newval) {
 711             if (!isBGPEntityOwner()) {
 712                 return;
 713             }
 714             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 715             synchronized (BgpConfigurationManager.this) {
 716                 int stalePathTime = newval.getStalepathTime().intValue();
 717                 BgpRouter br = getClient(YANG_OBJ);
 718                 if (br == null) {
 719                     LOG.error("{} Unable to update stale-path time to {}; {} {}", YANG_OBJ, stalePathTime,
 720                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 721                     return;
 722                 }
 723                 try {
 724                     br.addGracefulRestart(stalePathTime);
 725                 } catch (TException | BgpRouterException e) {
 726                     LOG.error("{} update received exception; {}", YANG_OBJ, ADD_WARN, e);
 727                 }
 728             }
 729         }
 730     }
 731
 732     public class LoggingReactor
 733             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 734             implements ClusteredDataTreeChangeListener<Logging> {
 735
 736         private static final String YANG_OBJ = "logging ";
 737
 738         public LoggingReactor() {
 739             super(Logging.class, LoggingReactor.class);
 740         }
 741
 742         @Override
 743         protected void add(InstanceIdentifier<Logging> iid, Logging val) {
 744             if (!isBGPEntityOwner()) {
 745                 return;
 746             }
 747             synchronized (BgpConfigurationManager.this) {
 748                 BgpRouter br = getClient(YANG_OBJ);
 749                 if (br == null) {
 750                     LOG.error("{} Unable to add logging for qbgp; {} {}", YANG_OBJ,
 751                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 752                     return;
 753                 }
 754                 try {
 755                     br.setLogging(val.getFile(), val.getLevel());
 756                 } catch (TException | BgpRouterException e) {
 757                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 758                 }
 759             }
 760         }
 761
 762         @Override
 763         protected LoggingReactor getDataTreeChangeListener() {
 764             return LoggingReactor.this;
 765         }
 766
 767         @Override
 768         protected InstanceIdentifier<Logging> getWildCardPath() {
 769             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 770         }
 771
 772         @Override
 773         protected void remove(InstanceIdentifier<Logging> iid, Logging val) {
 774             if (!isBGPEntityOwner()) {
 775                 return;
 776             }
 777             LOG.debug("received remove Logging config val {}", val.getLevel());
 778             synchronized (BgpConfigurationManager.this) {
 779                 BgpRouter br = getClient(YANG_OBJ);
 780                 if (br == null) {
 781                     LOG.error("{} Unable to remove logging for qbgp; {} {}", YANG_OBJ,
 782                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 783                     return;
 784                 }
 785                 try {
 786                     br.setLogging(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
 787                 } catch (TException | BgpRouterException e) {
 788                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 789                 }
 790             }
 791         }
 792
 793         @Override
 794         protected void update(InstanceIdentifier<Logging> iid,
 795                 Logging oldval, Logging newval) {
 796             if (!isBGPEntityOwner()) {
 797                 return;
 798             }
 799             synchronized (BgpConfigurationManager.this) {
 800                 BgpRouter br = getClient(YANG_OBJ);
 801                 if (br == null) {
 802                     LOG.error("{} Unable to update logging for qbgp; {} {}", YANG_OBJ,
 803                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 804                     return;
 805                 }
 806                 try {
 807                     br.setLogging(newval.getFile(), newval.getLevel());
 808                 } catch (TException | BgpRouterException e) {
 809                     LOG.error("{} newval received exception; {}", YANG_OBJ, ADD_WARN, e);
 810                 }
 811             }
 812         }
 813     }
 814
 815     public class NeighborsReactor
 816             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 817             implements ClusteredDataTreeChangeListener<Neighbors> {
 818
 819         private static final String YANG_OBJ = "neighbors ";
 820
 821         public NeighborsReactor() {
 822             super(Neighbors.class, NeighborsReactor.class);
 823         }
 824
 825         @Override
 826         protected void add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 827             if (!isBGPEntityOwner()) {
 828                 return;
 829             }
 830             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 831             synchronized (BgpConfigurationManager.this) {
 832                 String peerIp = val.getAddress().getValue();
 833                 long as = val.getRemoteAs();
 834                 final String md5Secret = extractMd5Secret(val);
 835                 BgpRouter br = getClient(YANG_OBJ);
 836                 if (br == null) {
 837                     LOG.debug("{} Unable to process add for peer {} as {}; {} {}", YANG_OBJ, peerIp, as,
 838                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 839                     return;
 840                 }
 841                 try {
 842                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 843                     br.addNeighbor(peerIp, as, md5Secret);
 844
 845                 } catch (TException | BgpRouterException e) {
 846                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 847                 }
 848             }
 849         }
 850
 851         @Override
 852         protected NeighborsReactor getDataTreeChangeListener() {
 853             return NeighborsReactor.this;
 854         }
 855
 856         @Override
 857         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 858             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class);
 859         }
 860
 861         @Override
 862         protected void remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 863             if (!isBGPEntityOwner()) {
 864                 return;
 865             }
 866             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 867             synchronized (BgpConfigurationManager.this) {
 868                 String peerIp = val.getAddress().getValue();
 869                 BgpRouter br = getClient(YANG_OBJ);
 870                 if (br == null) {
 871                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 872                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 873                     return;
 874                 }
 875                 try {
 876                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 877                     br.delNeighbor(peerIp);
 878                 } catch (TException | BgpRouterException e) {
 879                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 880                 }
 881
 882                 final BgpAlarms bgpAlarms = getBgpAlarms();
 883                 if (bgpAlarms != null) {
 884                     bgpAlarms.clearBgpNbrDownAlarm(peerIp);
 885                 }
 886             }
 887         }
 888
 889         @Override
 890         protected void update(InstanceIdentifier<Neighbors> iid,
 891                 Neighbors oldval, Neighbors newval) {
 892             if (!isBGPEntityOwner()) {
 893                 return;
 894             }
 895             //purposefully nothing to do.
 896         }
 897     }
 898
 899     public class EbgpMultihopReactor
 900             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 901             implements ClusteredDataTreeChangeListener<EbgpMultihop> {
 902
 903         private static final String YANG_OBJ = "ebgp-multihop ";
 904
 905         public EbgpMultihopReactor() {
 906             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 907         }
 908
 909         @Override
 910         protected void add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 911             if (!isBGPEntityOwner()) {
 912                 return;
 913             }
 914             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 915             synchronized (BgpConfigurationManager.this) {
 916                 String peerIp = val.getPeerIp().getValue();
 917                 BgpRouter br = getClient(YANG_OBJ);
 918                 if (br == null) {
 919                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
 920                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 921                     return;
 922                 }
 923                 try {
 924                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 925                 } catch (TException | BgpRouterException e) {
 926                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 927                 }
 928             }
 929         }
 930
 931         @Override
 932         protected EbgpMultihopReactor getDataTreeChangeListener() {
 933             return EbgpMultihopReactor.this;
 934         }
 935
 936         @Override
 937         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
 938             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(EbgpMultihop.class);
 939         }
 940
 941         @Override
 942         protected void remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 943             if (!isBGPEntityOwner()) {
 944                 return;
 945             }
 946             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 947             synchronized (BgpConfigurationManager.this) {
 948                 String peerIp = val.getPeerIp().getValue();
 949                 BgpRouter br = getClient(YANG_OBJ);
 950                 if (br == null) {
 951                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 952                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 953                     return;
 954                 }
 955                 try {
 956                     br.delEbgpMultihop(peerIp);
 957                 } catch (TException | BgpRouterException e) {
 958                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 959                 }
 960             }
 961         }
 962
 963         @Override
 964         protected void update(InstanceIdentifier<EbgpMultihop> iid,
 965                 EbgpMultihop oldval, EbgpMultihop newval) {
 966             if (!isBGPEntityOwner()) {
 967                 return;
 968             }
 969             LOG.error(YANG_OBJ + UPD_WARN);
 970         }
 971     }
 972
 973     public class UpdateSourceReactor
 974             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
 975             implements ClusteredDataTreeChangeListener<UpdateSource> {
 976
 977         private static final String YANG_OBJ = "update-source ";
 978
 979         public UpdateSourceReactor() {
 980             super(UpdateSource.class, UpdateSourceReactor.class);
 981         }
 982
 983         @Override
 984         protected void add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
 985             if (!isBGPEntityOwner()) {
 986                 return;
 987             }
 988             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
 989             synchronized (BgpConfigurationManager.this) {
 990                 String peerIp = val.getPeerIp().getValue();
 991                 BgpRouter br = getClient(YANG_OBJ);
 992                 if (br == null) {
 993                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
 994                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 995                     return;
 996                 }
 997                 try {
 998                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
 999                 } catch (TException | BgpRouterException e) {
1000                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1001                 }
1002             }
1003         }
1004
1005         @Override
1006         protected UpdateSourceReactor getDataTreeChangeListener() {
1007             return UpdateSourceReactor.this;
1008         }
1009
1010         @Override
1011         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
1012             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(UpdateSource.class);
1013         }
1014
1015         @Override
1016         protected void remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
1017             if (!isBGPEntityOwner()) {
1018                 return;
1019             }
1020             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
1021             synchronized (BgpConfigurationManager.this) {
1022                 String peerIp = val.getPeerIp().getValue();
1023                 BgpRouter br = getClient(YANG_OBJ);
1024                 if (br == null) {
1025                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1026                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1027                     return;
1028                 }
1029                 try {
1030                     br.delUpdateSource(peerIp);
1031                 } catch (TException | BgpRouterException e) {
1032                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1033                 }
1034             }
1035         }
1036
1037         @Override
1038         protected void update(InstanceIdentifier<UpdateSource> iid,
1039                 UpdateSource oldval, UpdateSource newval) {
1040             if (!isBGPEntityOwner()) {
1041                 return;
1042             }
1043             LOG.error(YANG_OBJ + UPD_WARN);
1044         }
1045     }
1046
1047     public class AddressFamiliesReactor
1048             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
1049             implements ClusteredDataTreeChangeListener<AddressFamilies> {
1050
1051         private static final String YANG_OBJ = "address-families ";
1052
1053         public AddressFamiliesReactor() {
1054             super(AddressFamilies.class, AddressFamiliesReactor.class);
1055         }
1056
1057         @Override
1058         protected void add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1059             if (!isBGPEntityOwner()) {
1060                 return;
1061             }
1062             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
1063             synchronized (BgpConfigurationManager.this) {
1064                 String peerIp = val.getPeerIp().getValue();
1065                 BgpRouter br = getClient(YANG_OBJ);
1066                 if (br == null) {
1067                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1068                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1069                     return;
1070                 }
1071                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1072                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1073                 try {
1074                     br.addAddressFamily(peerIp, afi, safi);
1075                 } catch (TException | BgpRouterException e) {
1076                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1077                 }
1078             }
1079         }
1080
1081         @Override
1082         protected AddressFamiliesReactor getDataTreeChangeListener() {
1083             return AddressFamiliesReactor.this;
1084         }
1085
1086         @Override
1087         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
1088             return InstanceIdentifier.create(Bgp.class).child(Neighbors.class).child(AddressFamilies.class);
1089         }
1090
1091         @Override
1092         protected void remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1093             if (!isBGPEntityOwner()) {
1094                 return;
1095             }
1096             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
1097             synchronized (BgpConfigurationManager.this) {
1098                 String peerIp = val.getPeerIp().getValue();
1099                 BgpRouter br = getClient(YANG_OBJ);
1100                 if (br == null) {
1101                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1102                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1103                     return;
1104                 }
1105                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1106                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1107                 try {
1108                     br.delAddressFamily(peerIp, afi, safi);
1109                 } catch (TException | BgpRouterException e) {
1110                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1111                 }
1112             }
1113         }
1114
1115         @Override
1116         protected void update(InstanceIdentifier<AddressFamilies> iid,
1117                 AddressFamilies oldval, AddressFamilies newval) {
1118             if (!isBGPEntityOwner()) {
1119                 return;
1120             }
1121             LOG.error(YANG_OBJ + UPD_WARN);
1122         }
1123     }
1124
1125     public class NetworksReactor
1126             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
1127             implements ClusteredDataTreeChangeListener<Networks> {
1128
1129         private static final String YANG_OBJ = "networks ";
1130
1131         public NetworksReactor() {
1132             super(Networks.class, NetworksReactor.class);
1133         }
1134
1135         @Override
1136         public NetworksReactor getDataTreeChangeListener() {
1137             return NetworksReactor.this;
1138         }
1139
1140         @Override
1141         protected void add(InstanceIdentifier<Networks> iid, Networks val) {
1142             if (!isBGPEntityOwner()) {
1143                 return;
1144             }
1145             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1146             synchronized (BgpConfigurationManager.this) {
1147                 String rd = val.getRd();
1148                 String pfxlen = val.getPrefixLen();
1149                 String nh = val.getNexthop().getValue();
1150                 BgpRouter br = getClient(YANG_OBJ);
1151                 if (br == null) {
1152                     LOG.debug("{} Unable to process add for rd {} prefix {} nexthop {}; {} {}", YANG_OBJ, rd, pfxlen,
1153                             nh, BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1154                     return;
1155                 }
1156                 Long label = val.getLabel();
1157                 int lbl = label == null ? qbgpConstants.LBL_NO_LABEL
1158                         : label.intValue();
1159                 int l3vni = val.getL3vni() == null ? qbgpConstants.LBL_NO_LABEL
1160                         : val.getL3vni().intValue();
1161                 int l2vni = val.getL2vni() == null ? qbgpConstants.LBL_NO_LABEL
1162                         : val.getL2vni().intValue();
1163
1164                 BgpControlPlaneType protocolType = val.getBgpControlPlaneType();
1165                 int ethernetTag = val.getEthtag().intValue();
1166                 String esi = val.getEsi();
1167                 String macaddress = val.getMacaddress();
1168                 EncapType encapType = val.getEncapType();
1169                 String routerMac = val.getRoutermac();
1170
1171                 try {
1172                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni, BgpUtil.convertToThriftProtocolType(protocolType),
1173                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1174                 } catch (TException | BgpRouterException e) {
1175                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1176                 }
1177             }
1178         }
1179
1180         @Override
1181         protected InstanceIdentifier<Networks> getWildCardPath() {
1182             return InstanceIdentifier.create(Bgp.class).child(Networks.class);
1183         }
1184
1185         @Override
1186         protected void remove(InstanceIdentifier<Networks> iid, Networks val) {
1187             if (!isBGPEntityOwner()) {
1188                 return;
1189             }
1190             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1191             synchronized (BgpConfigurationManager.this) {
1192                 String rd = val.getRd();
1193                 String pfxlen = val.getPrefixLen();
1194                 BgpRouter br = getClient(YANG_OBJ);
1195                 if (br == null) {
1196                     LOG.debug("{} Unable to process remove for rd {} prefix {}; {} {}", YANG_OBJ, rd, pfxlen,
1197                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1198                     return;
1199                 }
1200                 Long label = val.getLabel();
1201                 int lbl = label == null ? 0 : label.intValue();
1202                 if (rd == null && lbl > 0) {
1203                     //LU prefix is being deleted.
1204                     rd = Integer.toString(lbl);
1205                 }
1206                 try {
1207                     br.delPrefix(rd, pfxlen);
1208                 } catch (TException | BgpRouterException e) {
1209                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1210                 }
1211             }
1212         }
1213
1214         /**get the value AFI from a prefix as "x.x.x.x/x".
1215          *
1216          * @param pfxlen the prefix to get an afi
1217          * @return the afi value as you are need
1218          */
1219         public  int  testValueAFI(String pfxlen) {
1220             int afiNew = af_afi.AFI_IP.getValue();
1221             try {
1222                 String ipOnly = pfxlen.substring(0, pfxlen.lastIndexOf("/"));
1223                 java.net.Inet6Address.getByName(ipOnly);
1224                 afiNew = af_afi.AFI_IPV6.getValue();
1225             } catch (java.net.UnknownHostException e) {
1226                 //ce n'est pas de l'ipv6
1227             }
1228             return afiNew;
1229         }
1230
1231
1232         @Override
1233         protected void update(final InstanceIdentifier<Networks> iid,
1234                 final Networks oldval, final Networks newval) {
1235             if (!isBGPEntityOwner()) {
1236                 return;
1237             }
1238             if (oldval.equals(newval)) {
1239                 //Update: OLD and New values are same, no need to trigger remove/add.
1240                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1241                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1242                 return;
1243             }
1244             LOG.debug("update networks old val RD: {}, Prefix: {}, Label: {}, NH: {} "
1245                             + "new val RD: {}, Prefix: {}, Label: {}, NH: {}",
1246                     oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop(),
1247                     newval.getRd(), newval.getPrefixLen(), newval.getLabel(), newval.getNexthop());
1248             remove(iid, oldval);
1249             add(iid, newval);
1250         }
1251     }
1252
1253     static Timer timer = new Timer();
1254
1255     public class VrfsReactor
1256             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1257             implements ClusteredDataTreeChangeListener<Vrfs> {
1258
1259         private static final String YANG_OBJ = "vrfs ";
1260
1261         public VrfsReactor() {
1262             super(Vrfs.class, VrfsReactor.class);
1263         }
1264
1265         @Override
1266         protected void add(InstanceIdentifier<Vrfs> iid, Vrfs vrfs) {
1267             if (!isBGPEntityOwner()) {
1268                 return;
1269             }
1270             LOG.debug("received add Vrfs config value {}", vrfs.getRd());
1271             synchronized (BgpConfigurationManager.this) {
1272                 String rd = vrfs.getRd();
1273                 BgpRouter br = getClient(YANG_OBJ);
1274                 if (br == null) {
1275                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1276                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1277                     return;
1278                 }
1279                 try {
1280                     List<AddressFamiliesVrf> vrfAddrFamilyList = vrfs.getAddressFamiliesVrf();
1281                     for (AddressFamiliesVrf vrfAddrFamily : vrfAddrFamilyList) {
1282                         /*add to br the new vrfs arguments*/
1283                         br.addVrf(BgpUtil.getLayerType(vrfAddrFamily), rd, vrfs.getImportRts(), vrfs.getExportRts()
1284                         );
1285                     }
1286                     /*add to br the vrfs contained in mapNewAdFamily*/
1287                     List<AddressFamiliesVrf> vrfAddrFamilyListFromMap = mapNewAdFamily.get(rd);
1288                     if (vrfAddrFamilyListFromMap == null) {
1289                         return;
1290                     }
1291
1292                     for (AddressFamiliesVrf adf : vrfAddrFamilyListFromMap) {
1293                         if (vrfAddrFamilyList.contains(adf)) {
1294                             mapNewAdFamily.remove(rd);
1295                         } else  if (adf != null) {
1296
1297                             br.addVrf(BgpUtil.getLayerType(adf), rd, vrfs.getImportRts(), vrfs.getExportRts()
1298                             );
1299                             // remove AddressFamiliesVrf which was already added to BGP
1300                             vrfAddrFamilyListFromMap.remove(adf);
1301                             if (vrfAddrFamilyListFromMap.isEmpty()) {
1302                                 // remove Vrf entry from temp mapNewAdFamily if all its AddressFamiliesVrf was
1303                                 // added to BGP
1304                                 mapNewAdFamily.remove(rd);
1305                             }
1306                         }
1307                     }
1308                 } catch (TException | BgpRouterException e) {
1309                     LOG.error("{} get {}, Add received exception", YANG_OBJ, ADD_WARN, e);
1310                 }
1311             }
1312         }
1313
1314         @Override
1315         protected VrfsReactor getDataTreeChangeListener() {
1316             return VrfsReactor.this;
1317         }
1318
1319         @Override
1320         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1321             return InstanceIdentifier.create(Bgp.class).child(Vrfs.class);
1322         }
1323
1324         @Override
1325         protected void remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1326             if (!isBGPEntityOwner()) {
1327                 return;
1328             }
1329             LOG.debug("received remove Vrfs config val {}", val.getRd());
1330             synchronized (BgpConfigurationManager.this) {
1331                 String rd = val.getRd();
1332                 BgpRouter br = getClient(YANG_OBJ);
1333                 if (br == null) {
1334                     LOG.debug("{} Unable to process remove for rd {}; {} {}", YANG_OBJ, rd,
1335                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1336                     return;
1337                 }
1338                 try {
1339                     List<AddressFamiliesVrf> adf = mapNewAdFamily.get(rd);
1340                     adf = adf != null ? adf : new ArrayList<>();
1341                     for (AddressFamiliesVrf s : val.getAddressFamiliesVrf()) {
1342                         br.delVrf(rd, s.getAfi(), s.getSafi());
1343                         adf.remove(s);// remove in the map the vrf in waiting for advertise quagga
1344                     }
1345                     if (adf.isEmpty()) {
1346                         mapNewAdFamily.remove(rd);
1347                     }
1348                 } catch (TException | BgpRouterException e) {
1349                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1350                 }
1351             }
1352         }
1353
1354         @Override
1355         protected void update(InstanceIdentifier<Vrfs> iid,
1356                 Vrfs oldval, Vrfs newval) {
1357             if (oldval != null && newval != null) {
1358                 LOG.debug("received update Vrfs config val {}, VRFS: Update getting triggered for VRFS rd {}",
1359                         newval.getRd(), oldval.getRd());
1360             } else {
1361                 LOG.debug("received update Vrfs config val {}, from old vrf {}",
1362                         newval, oldval);
1363             }
1364             if (!isBGPEntityOwner()) {
1365                 return;
1366             }
1367
1368             List<AddressFamiliesVrf> adFamilyVrfToDel = new ArrayList<>();
1369             List<AddressFamiliesVrf> adFamilyVrfToAdd = new ArrayList<>();
1370             List<AddressFamiliesVrf> oldlistAdFamilies = new ArrayList<>();
1371             List<AddressFamiliesVrf> newlistAdFamilies = new ArrayList<>();
1372             if (oldval != null) {
1373                 oldlistAdFamilies = oldval.getAddressFamiliesVrf() == null
1374                         ? new ArrayList<>() : oldval.getAddressFamiliesVrf();
1375             }
1376             if (newval != null) {
1377                 newlistAdFamilies = newval.getAddressFamiliesVrf() == null
1378                         ? new ArrayList<>() : newval.getAddressFamiliesVrf();
1379             }
1380             /*find old AddressFamily to remove from new configuration*/
1381             for (AddressFamiliesVrf adVrf : oldlistAdFamilies) {
1382                 if (!newlistAdFamilies.contains(adVrf)) {
1383                     adFamilyVrfToDel.add(adVrf);
1384                 }
1385             }
1386             /*find new AddressFamily to add to unexisting configuration*/
1387             for (AddressFamiliesVrf adVrf : newlistAdFamilies) {
1388                 if (!oldlistAdFamilies.contains(adVrf)) {
1389                     adFamilyVrfToAdd.add(adVrf);
1390                 }
1391             }
1392             String rd = newval != null ? newval.getRd() : null;
1393             if (rd != null) {
1394                 BgpRouter br = getClient(YANG_OBJ);
1395                 if (br == null) {
1396                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1397                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1398                     return;
1399                 }
1400
1401                 for (AddressFamiliesVrf adfvrf : adFamilyVrfToAdd) {
1402                     try {
1403                         LOG.debug("call addVRf rd {} afi {} safi {}", rd, adfvrf.getAfi(), adfvrf.getSafi());
1404                         br.addVrf(BgpUtil.getLayerType(adfvrf), rd, newval.getImportRts(),
1405                                 newval.getExportRts());
1406                     } catch (TException | BgpRouterException e) {
1407                         LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1408                     }
1409                 }
1410
1411                 for (AddressFamiliesVrf adfToDel : adFamilyVrfToDel) {
1412                     try {
1413                         LOG.debug("call delVRf rd {} afi {} safi {}", rd, adfToDel.getAfi(), adfToDel.getSafi());
1414                         br.delVrf(rd, adfToDel.getAfi(), adfToDel.getSafi());
1415                     } catch (TException | BgpRouterException e) {
1416                         LOG.error("{} delVrf received exception; {}", YANG_OBJ, ADD_WARN, e);
1417                     }
1418                 }
1419             }
1420         }
1421     }
1422
1423     protected void activateMIP() {
1424         try {
1425             LOG.trace("BgpReactor: Executing MIP Activate command");
1426             Runtime.getRuntime().exec("cluster ip -a sdnc_bgp_mip");
1427             Runtime.getRuntime().exec("cluster ip -a sdnc_os_mip");
1428             LOG.trace("bgpMIP Activated");
1429
1430         } catch (IOException io) {
1431             LOG.error("IO Exception got while activating mip: {}", io.getMessage());
1432         }
1433     }
1434
1435     public class BgpReactor
1436             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1437             implements ClusteredDataTreeChangeListener<Bgp> {
1438
1439         private static final String YANG_OBJ = "Bgp ";
1440
1441         public BgpReactor() {
1442             super(Bgp.class, BgpReactor.class);
1443         }
1444
1445
1446         @Override
1447         protected void add(InstanceIdentifier<Bgp> iid, Bgp val) {
1448             LOG.debug("received add Bgp config");
1449
1450             try {
1451                 initer.await();
1452             } catch (InterruptedException e) {
1453                 // Ignored
1454             }
1455             synchronized (BgpConfigurationManager.this) {
1456                 config = val;
1457                 if (!isBGPEntityOwner()) {
1458                     return;
1459                 }
1460                 activateMIP();
1461             }
1462         }
1463
1464         @Override
1465         protected BgpReactor getDataTreeChangeListener() {
1466             return BgpReactor.this;
1467         }
1468
1469         @Override
1470         protected InstanceIdentifier<Bgp> getWildCardPath() {
1471             return InstanceIdentifier.create(Bgp.class);
1472         }
1473
1474         @Override
1475         protected void remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1476             if (!isBGPEntityOwner()) {
1477                 return;
1478             }
1479             LOG.debug("received remove Bgp config");
1480
1481             config = null;
1482         }
1483
1484         @Override
1485         protected void update(InstanceIdentifier<Bgp> iid,
1486                 Bgp oldval, Bgp newval) {
1487             if (!isBGPEntityOwner()) {
1488                 return;
1489             }
1490
1491             config = newval;
1492         }
1493     }
1494
1495     @SuppressWarnings("deprecation")
1496     public class MultipathReactor
1497             extends AsyncDataTreeChangeListenerBase<Multipath, MultipathReactor>
1498             implements ClusteredDataTreeChangeListener<Multipath> {
1499
1500         private static final String YANG_OBJ = "multipath ";
1501
1502         public MultipathReactor() {
1503             super(Multipath.class, MultipathReactor.class);
1504         }
1505
1506
1507         @Override
1508         protected MultipathReactor getDataTreeChangeListener() {
1509             return MultipathReactor.this;
1510         }
1511
1512         @Override
1513         protected InstanceIdentifier<Multipath> getWildCardPath() {
1514             return InstanceIdentifier.create(Bgp.class).child(Multipath.class);
1515         }
1516
1517         @Override
1518         protected void remove(InstanceIdentifier<Multipath> iid, Multipath val) {
1519             executor.execute(new MultipathStatusChange(val));
1520         }
1521
1522         @Override
1523         protected void update(InstanceIdentifier<Multipath> iid, Multipath oldval, Multipath newval) {
1524             executor.execute(new MultipathStatusChange(newval));
1525         }
1526
1527         @Override
1528         protected void add(InstanceIdentifier<Multipath> key, Multipath dataObjectModification) {
1529             executor.execute(new MultipathStatusChange(dataObjectModification));
1530         }
1531
1532         class MultipathStatusChange implements Runnable {
1533
1534             Multipath multipath;
1535
1536             MultipathStatusChange(Multipath multipath) {
1537                 this.multipath = multipath;
1538             }
1539
1540             @Override
1541             public void run() {
1542                 if (isBGPEntityOwner()) {
1543                     synchronized (BgpConfigurationManager.this) {
1544
1545                         BgpRouter br = getClient(YANG_OBJ);
1546
1547                         if (br != null) {
1548                             af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1549                             af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1550
1551                             try {
1552                                 if (multipath.isMultipathEnabled()) {
1553                                     br.enableMultipath(afi, safi);
1554                                 } else {
1555                                     br.disableMultipath(afi, safi);
1556                                 }
1557                             } catch (TException | BgpRouterException e) {
1558                                 LOG.error("{} received exception", YANG_OBJ, e);
1559                             }
1560                         }
1561                     }
1562                 }
1563             }
1564
1565         }
1566
1567         @Override
1568         public void close() {
1569             super.close();
1570         }
1571     }
1572
1573     @SuppressWarnings("deprecation")
1574     public class VrfMaxpathReactor
1575             extends AsyncDataTreeChangeListenerBase<VrfMaxpath, VrfMaxpathReactor>
1576             implements ClusteredDataTreeChangeListener<VrfMaxpath> {
1577
1578         private static final String YANG_OBJ = "vrfMaxpath ";
1579
1580         public VrfMaxpathReactor() {
1581             super(VrfMaxpath.class, VrfMaxpathReactor.class);
1582         }
1583
1584
1585         @Override
1586         protected VrfMaxpathReactor getDataTreeChangeListener() {
1587             return VrfMaxpathReactor.this;
1588         }
1589
1590         @Override
1591         protected InstanceIdentifier<VrfMaxpath> getWildCardPath() {
1592             return InstanceIdentifier.create(Bgp.class).child(VrfMaxpath.class);
1593         }
1594
1595         class VrfMaxPathConfigurator implements Runnable {
1596
1597             VrfMaxpath vrfMaxpathVal;
1598
1599             VrfMaxPathConfigurator(VrfMaxpath vrfMaxPathVal) {
1600                 this.vrfMaxpathVal = vrfMaxPathVal;
1601             }
1602
1603             @Override
1604             public void run() {
1605                 if (isBGPEntityOwner()) {
1606                     synchronized (BgpConfigurationManager.this) {
1607                         BgpRouter br = getClient(YANG_OBJ);
1608                         if (br != null) {
1609                             try {
1610                                 br.multipaths(vrfMaxpathVal.getRd(), vrfMaxpathVal.getMaxpaths());
1611                                 LOG.debug("Maxpath for vrf {} is {}", vrfMaxpathVal.getRd(),
1612                                         vrfMaxpathVal.getMaxpaths());
1613                             } catch (TException | BgpRouterException e) {
1614                                 LOG.error("{} received exception", YANG_OBJ, e);
1615                             }
1616                         }
1617                     }
1618                 }
1619             }
1620         }
1621
1622         @Override
1623         protected void remove(InstanceIdentifier<VrfMaxpath> iid, VrfMaxpath vrfMaxPathVal) {
1624             executor.execute(new VrfMaxPathConfigurator(vrfMaxPathVal));
1625         }
1626
1627         @Override
1628         protected void update(InstanceIdentifier<VrfMaxpath> iid,
1629                               VrfMaxpath oldval, VrfMaxpath newval) {
1630             if (!Objects.equals(oldval.getMaxpaths(), newval.getMaxpaths())) {
1631                 executor.execute(new VrfMaxPathConfigurator(newval));
1632             }
1633         }
1634
1635         @Override
1636         protected void add(InstanceIdentifier<VrfMaxpath> instanceIdentifier, VrfMaxpath vrfMaxpathVal) {
1637             executor.execute(new VrfMaxPathConfigurator(vrfMaxpathVal));
1638         }
1639
1640         @Override
1641         public void close() {
1642             super.close();
1643         }
1644     }
1645
1646     public boolean isIpAvailable(String odlip) {
1647
1648         try {
1649             if (odlip != null) {
1650                 if ("127.0.0.1".equals(odlip)) {
1651                     return true;
1652                 }
1653                 Enumeration<NetworkInterface> networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces();
1654                 while (networkInterfaceEnumeration.hasMoreElements()) {
1655                     NetworkInterface networkInterface = networkInterfaceEnumeration.nextElement();
1656                     Enumeration<InetAddress>  inetAddressEnumeration = networkInterface.getInetAddresses();
1657                     while (inetAddressEnumeration.hasMoreElements()) {
1658                         InetAddress inetAddress = inetAddressEnumeration.nextElement();
1659                         if (odlip.equals(inetAddress.getHostAddress())) {
1660                             return true;
1661                         }
1662                     }
1663                 }
1664             }
1665         } catch (SocketException e) {
1666             // Ignored?
1667         }
1668         return false;
1669     }
1670
1671     public long getStalePathtime(int defValue, AsId asId) {
1672         long spt = 0;
1673         try {
1674             spt = getConfig().getGracefulRestart().getStalepathTime();
1675         } catch (NullPointerException e) {
1676             try {
1677                 spt = asId.getStalepathTime();
1678                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1679             } catch (NullPointerException ignore) {
1680                 LOG.trace("BGP AS id is not set using graceful");
1681                 spt = defValue;
1682             }
1683         }
1684         if (spt == 0) {
1685             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1686             spt = defValue;
1687         }
1688         return spt;
1689     }
1690
1691     public static boolean isValidConfigBgpHostPort(String bgpHost, int bgpPort) {
1692         if (!bgpHost.equals(DEF_CHOST)) {
1693             return true;
1694         } else {
1695             return false;
1696         }
1697     }
1698
1699     public synchronized void bgpRestarted() {
1700         /*
1701          * If there a thread which in the process of stale cleanup, cancel it
1702          * and start a new thread (to avoid processing same again).
1703          */
1704         if (previousReplayJobInProgress()) {
1705             cancelPreviousReplayJob();
1706         }
1707         Runnable task = () -> {
1708             LOG.info("running bgp replay task ");
1709             if (get() == null) {
1710                 String host = getConfigHost();
1711                 int port = getConfigPort();
1712                 LOG.info("connecting  to bgp host {} ", host);
1713                 bgpRouter.connect(host, port);
1714                 LOG.info("no config to push in bgp replay task ");
1715                 return;
1716             }
1717             setStaleStartTime(System.currentTimeMillis());
1718             LOG.info("started creating stale fibDSWriter  map ");
1719             createStaleFibMap();
1720             setStaleEndTime(System.currentTimeMillis());
1721             LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
1722             LOG.info("started bgp config replay ");
1723             setCfgReplayStartTime(System.currentTimeMillis());
1724             boolean replaySucceded = replay();
1725             setCfgReplayEndTime(System.currentTimeMillis());
1726             LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1727             if (replaySucceded) {
1728                 LOG.info("starting the stale cleanup timer");
1729                 long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1730                 setStaleCleanupTime(routeSyncTime);
1731                 routeCleanupFuture = executor.schedule(new RouteCleanup(), routeSyncTime, TimeUnit.SECONDS);
1732             } else {
1733                 staledFibEntriesMap.clear();
1734             }
1735         };
1736         lastReplayJobFt = executor.submit(task);
1737     }
1738
1739     private boolean previousReplayJobInProgress() {
1740         return lastReplayJobFt != null && !lastReplayJobFt.isDone();
1741     }
1742
1743     private void cancelPreviousReplayJob() {
1744         try {
1745             LOG.error("cancelling already running bgp replay task");
1746             if (lastReplayJobFt != null) {
1747                 lastReplayJobFt.cancel(true);
1748                 lastReplayJobFt = null;
1749                 staledFibEntriesMap.clear();
1750             }
1751             if (routeCleanupFuture != null) {
1752                 routeCleanupFuture.cancel(true);
1753                 routeCleanupFuture = null;
1754                 staledFibEntriesMap.clear();
1755             }
1756             Thread.sleep(2000);
1757         } catch (InterruptedException e) {
1758             LOG.error("Failed to cancel previous replay job ", e);
1759         }
1760     }
1761
1762     private void doRouteSync() {
1763         LOG.error("Starting BGP route sync");
1764         try {
1765             bgpRouter.initRibSync(bgpSyncHandle);
1766         } catch (BgpRouterException e) {
1767             LOG.error("Route sync aborted, exception when initializing", e);
1768             return;
1769         }
1770         while (bgpSyncHandle.getState() != BgpSyncHandle.DONE) {
1771             for (af_afi afi : af_afi.values()) {
1772                 Routes routes = null;
1773                 try {
1774                     routes = bgpRouter.doRibSync(bgpSyncHandle, afi);
1775                 } catch (TException | BgpRouterException e) {
1776                     LOG.error("Route sync aborted, exception when syncing", e);
1777                     return;
1778                 }
1779                 Iterator<Update> updates = routes.getUpdatesIterator();
1780                 while (updates.hasNext()) {
1781                     Update update = updates.next();
1782                     String rd = update.getRd();
1783                     String nexthop = update.getNexthop();
1784
1785                     // TODO: decide correct label here
1786                     int label = update.getL3label();
1787                     int l2label = update.getL2label();
1788
1789                     String prefix = update.getPrefix();
1790                     int plen = update.getPrefixlen();
1791
1792
1793                     // TODO: protocol type will not be available in "update"
1794                     // use "rd" to query vrf table and obtain the protocol_type.
1795                     // Currently using PROTOCOL_EVPN as default.
1796                     onUpdatePushRoute(
1797                            protocol_type.PROTOCOL_EVPN,
1798                            rd,
1799                            prefix,
1800                            plen,
1801                            nexthop,
1802                            update.getMacaddress(),
1803                            label,
1804                            l2label,
1805                            update.getRoutermac(),
1806                            afi);
1807                 }
1808             }
1809         }
1810         try {
1811             LOG.error("Ending BGP route-sync");
1812             bgpRouter.endRibSync(bgpSyncHandle);
1813         } catch (BgpRouterException e) {
1814             // Ignored?
1815         }
1816     }
1817
1818     public void addTepToElanDS(String rd, String tepIp, String mac, Long l2vni) {
1819         boolean needUpdate = addToRt2TepMap(rd, tepIp, mac, l2vni);
1820         if (needUpdate) {
1821             LOG.info("Adding tepIp {} with RD {} to ELan DS", tepIp, rd);
1822             bgpUtil.addTepToElanInstance(rd, tepIp);
1823         } else {
1824             LOG.debug("Skipping the Elan update for RT2 from tep {} rd {}", tepIp, rd);
1825         }
1826     }
1827
1828     public void deleteTepfromElanDS(String rd, String tepIp, String mac) {
1829         boolean needUpdate = deleteFromRt2TepMap(rd, tepIp, mac);
1830         if (needUpdate) {
1831             LOG.info("Deleting tepIp {} with RD {} to ELan DS", tepIp, rd);
1832             bgpUtil.deleteTepFromElanInstance(rd, tepIp);
1833         } else {
1834             LOG.debug("Skipping the Elan update for RT2 withdraw from tep {} rd {}", tepIp, rd);
1835         }
1836     }
1837
1838     /* onUpdatePushRoute
1839      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1840      *  - Entry compare shall include NextHop, Label.
1841      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1842      *  - If entry not found, add to FIB Config DS.
1843      *  - If entry found, but either Label/NextHop doesn't match.
1844      *      - Update FIB Config DS with modified values.
1845      *      - delete from Stale Map.
1846      */
1847
1848     public void onUpdatePushRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
1849                                   String macaddress, int label, int l2label, String routermac, af_afi afi) {
1850         boolean addroute = false;
1851         boolean macupdate = false;
1852         long l3vni = 0L;
1853         VrfEntry.EncapType encapType = VrfEntry.EncapType.Mplsgre;
1854         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1855             encapType = VrfEntry.EncapType.Vxlan;
1856             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
1857             if (vpnInstanceOpDataEntry != null) {
1858                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
1859                     LOG.info("Got RT2 route for RD {} l3label {} l2label {} from tep {} with mac {} remote RD {}",
1860                             vpnInstanceOpDataEntry.getVpnInstanceName(), label, l2label, nextHop, macaddress, rd);
1861                     addTepToElanDS(rd, nextHop, macaddress, (long)l2label);
1862                     macupdate = true;
1863                 } else {
1864                     l3vni = vpnInstanceOpDataEntry.getL3vni();
1865                 }
1866             } else {
1867                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1868                 return;
1869             }
1870         }
1871
1872         if (!staledFibEntriesMap.isEmpty()) {
1873             // restart Scenario, as MAP is not empty.
1874             Map<String, Long> map = staledFibEntriesMap.get(rd);
1875             if (map != null) {
1876                 String prefixNextHop = appendNextHopToPrefix(prefix + "/" + plen, nextHop);
1877                 Long labelInStaleMap = map.get(prefixNextHop);
1878                 if (null == labelInStaleMap) {
1879                     // New Entry, which happened to be added during restart.
1880                     addroute = true;
1881                 } else {
1882                     map.remove(prefixNextHop);
1883                     if (isRouteModified(label, labelInStaleMap)) {
1884                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1885                         // Existing entry, where in Label got modified during restart
1886                         addroute = true;
1887                     }
1888                 }
1889             } else {
1890                 LOG.debug("rd {} map is null while processing prefix {} ", rd, prefix);
1891                 addroute = true;
1892             }
1893         } else {
1894             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1895             addroute = true;
1896         }
1897         if (macupdate) {
1898             LOG.info("ADD: Adding Mac Fib entry rd {} mac{} nexthop {} l2vni {}", rd, macaddress, nextHop, l2label);
1899             fibDSWriter.addMacEntryToDS(rd, macaddress, prefix, Collections.singletonList(nextHop),
1900                     encapType, l2label, routermac, RouteOrigin.BGP);
1901             LOG.info("ADD: Added Mac Fib entry rd {} prefix {} nexthop {} label {}", rd, macaddress, nextHop, l2label);
1902         } else if (addroute) {
1903             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {} afi {}",
1904                     rd, prefix, nextHop, label, afi);
1905             // TODO: modify addFibEntryToDS signature
1906             List<String> nextHopList = Collections.singletonList(nextHop);
1907             fibDSWriter.addFibEntryToDS(rd, prefix + "/" + plen, nextHopList, encapType, label, l3vni,
1908                                         routermac, RouteOrigin.BGP);
1909             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1910             String vpnName = bgpUtil.getVpnNameFromRd(rd);
1911             if (vpnName != null) {
1912                 vpnLinkService.leakRouteIfNeeded(vpnName, prefix, nextHopList, label, RouteOrigin.BGP,
1913                                                  NwConstants.ADD_FLOW);
1914             }
1915         }
1916     }
1917
1918     public void onUpdateWithdrawRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
1919             String macaddress) {
1920         long vni = 0L;
1921         boolean macupdate = false;
1922         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1923             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
1924             if (vpnInstanceOpDataEntry != null) {
1925                 vni = vpnInstanceOpDataEntry.getL3vni();
1926                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
1927                     LOG.debug("Got RT2 withdraw for RD {} {} from tep {} with mac {} remote RD {}",
1928                             vpnInstanceOpDataEntry.getVpnInstanceName(), vni, nextHop, macaddress, rd);
1929                     deleteTepfromElanDS(rd, nextHop, macaddress);
1930                     LOG.debug("For rd {}. skipping fib update", rd);
1931                     macupdate = true;
1932                 }
1933             } else {
1934                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1935                 return;
1936             }
1937         }
1938         if (macupdate) {
1939             LOG.info("Removing Mac Fib entry rd {} mac{} nexthop {} ", rd, macaddress, nextHop);
1940             fibDSWriter.removeMacEntryFromDS(rd, macaddress);
1941             LOG.info("Removed Mac Fib entry rd {} prefix {} nexthop {} ", rd, macaddress, nextHop);
1942         } else {
1943             LOG.info("REMOVE: Removing Fib entry rd {} prefix {}", rd, prefix);
1944             fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nextHop);
1945             LOG.info("REMOVE: Removed Fib entry rd {} prefix {}", rd, prefix);
1946         }
1947     }
1948
1949     //TODO: below function is for testing purpose with cli
1950     public void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
1951         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
1952         fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nexthop);
1953         String vpnName = bgpUtil.getVpnNameFromRd(rd);
1954         if (vpnName != null) {
1955             vpnLinkService.leakRouteIfNeeded(vpnName, prefix, null /*nextHopList*/, 0 /*INVALID_LABEL*/,
1956                                              RouteOrigin.BGP, NwConstants.DEL_FLOW);
1957         }
1958     }
1959
1960     private static boolean isRouteModified(int label, Long labelInStaleMap) {
1961         return labelInStaleMap != null && !labelInStaleMap.equals(Long.valueOf(label));
1962     }
1963
1964     static class ReplayNbr {
1965         Neighbors nbr;
1966         boolean shouldRetry = false;
1967
1968         public Neighbors getNbr() {
1969             return nbr;
1970         }
1971
1972         public boolean isShouldRetry() {
1973             return shouldRetry;
1974         }
1975
1976         public void setShouldRetry(boolean retryNbr) {
1977             this.shouldRetry = retryNbr;
1978         }
1979
1980         ReplayNbr(Neighbors nbr, boolean shouldRetry) {
1981             this.nbr = nbr;
1982             this.shouldRetry = shouldRetry;
1983         }
1984     }
1985
1986     private static boolean replayNbrConfig(List<Neighbors> neighbors, BgpRouter br) {
1987         if (neighbors == null || neighbors.isEmpty()) {
1988             LOG.error("Replaying nbr configuration, received NULL list ");
1989             return true;
1990         }
1991
1992         List<ReplayNbr> replayNbrList = new ArrayList<>();
1993         for (Neighbors nbr : neighbors) {
1994             if (nbr != null) {
1995                 replayNbrList.add(new ReplayNbr(nbr, true));
1996             }
1997         }
1998         final int numberOfNbrRetries = 3;
1999         RetryOnException nbrRetry = new RetryOnException(numberOfNbrRetries);
2000         do {
2001             for (ReplayNbr replayNbr : replayNbrList) {
2002                 if (!replayNbr.isShouldRetry()) {
2003                     continue;
2004                 }
2005                 boolean replayDone = false;
2006                 LOG.debug("Replaying addNbr {}", replayNbr.getNbr().getAddress().getValue());
2007                 replayDone = false;
2008                 try {
2009                     final String md5password = extractMd5Secret(replayNbr.getNbr());
2010                     br.addNeighbor(replayNbr.getNbr().getAddress().getValue(),
2011                             replayNbr.getNbr().getRemoteAs().longValue(), md5password);
2012                     UpdateSource us = replayNbr.getNbr().getUpdateSource();
2013                     if (us != null) {
2014                         LOG.debug("Replaying updatesource along with nbr: {} US-ip: {} to peer {}",
2015                                 replayNbr.getNbr().getAddress().getValue(),
2016                                 us.getSourceIp().getValue(),
2017                                 us.getPeerIp().getValue());
2018                         br.addUpdateSource(us.getPeerIp().getValue(),
2019                                 us.getSourceIp().getValue());
2020                     }
2021                     replayDone = true;
2022                 } catch (TException | BgpRouterException eNbr) {
2023                     LOG.debug("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), eNbr);
2024                 }
2025                 boolean replaySuccess = true;
2026                 replaySuccess = replaySuccess && replayDone;
2027                 LOG.debug("Replay addNbr {} successful", replayNbr.getNbr().getAddress().getValue());
2028
2029                 //Update Source handling
2030                 UpdateSource us = replayNbr.getNbr().getUpdateSource();
2031                 if (replayDone == false && us != null) {
2032                     LOG.debug("Replaying updatesource {} to peer {}", us.getSourceIp().getValue(),
2033                             us.getPeerIp().getValue());
2034                     replayDone = false;
2035                     try {
2036                         br.addUpdateSource(us.getPeerIp().getValue(),
2037                                 us.getSourceIp().getValue());
2038                         replayDone = true;
2039                     } catch (TException | BgpRouterException eUs) {
2040                         LOG.debug("Replaying UpdateSource for Nbr {}, exception:",
2041                                 replayNbr.getNbr().getAddress().getValue(), eUs);
2042                     }
2043                     LOG.debug("Replay updatesource {} successful", us.getSourceIp().getValue());
2044                     replaySuccess = replaySuccess && replayDone;
2045                 }
2046                 //Ebgp Multihope
2047                 EbgpMultihop en = replayNbr.getNbr().getEbgpMultihop();
2048                 if (en != null) {
2049                     replayDone = false;
2050                     try {
2051                         br.addEbgpMultihop(en.getPeerIp().getValue(),
2052                                 en.getNhops().intValue());
2053                         replayDone = true;
2054                     } catch (TException | BgpRouterException eEbgpMhop) {
2055                         LOG.debug("Replaying EbgpMultihop for Nbr {}, exception: ",
2056                                 replayNbr.getNbr().getAddress().getValue(), eEbgpMhop);
2057                     }
2058                     replaySuccess = replaySuccess && replayDone;
2059                 }
2060
2061                 //afs
2062                 List<AddressFamilies> afs = replayNbr.getNbr().getAddressFamilies();
2063                 if (afs != null) {
2064                     for (AddressFamilies af : afs) {
2065                         af_afi afi = af_afi.findByValue(af.getAfi().intValue());
2066                         af_safi safi = af_safi.findByValue(af.getSafi().intValue());
2067                         replayDone = false;
2068                         try {
2069                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
2070                             replayDone = true;
2071                         } catch (TException | BgpRouterException eAFs) {
2072                             LOG.debug("Replaying AddressFamily for Nbr {}, exception:",
2073                                     replayNbr.getNbr().getAddress().getValue(), eAFs);
2074                         }
2075                         replaySuccess = replaySuccess && replayDone;
2076                     }
2077                 }
2078                 //replay is success --> no need to replay this nbr in next iteration.
2079                 replayNbr.setShouldRetry(replaySuccess ? false : true);
2080             }
2081         } while (nbrRetry.decrementAndRetry());
2082         boolean replaySuccess = true;
2083         for (ReplayNbr replayNbr : replayNbrList) {
2084             replaySuccess = replaySuccess && !replayNbr.isShouldRetry();
2085         }
2086         return replaySuccess;
2087     }
2088
2089     public String getConfigHost() {
2090         if (config == null) {
2091             return hostStartup;
2092         }
2093         ConfigServer ts = config.getConfigServer();
2094         return ts == null ? hostStartup : ts.getHost().getValue();
2095     }
2096
2097     public int getConfigPort() {
2098         if (config == null) {
2099             return Integer.parseInt(portStartup);
2100         }
2101         ConfigServer ts = config.getConfigServer();
2102         return ts == null ? Integer.parseInt(portStartup) :
2103                 ts.getPort().intValue();
2104     }
2105
2106     public Bgp getConfig() {
2107         AtomicInteger bgpDSretryCount = new AtomicInteger(DS_RETRY_COUNT);
2108         while (0 != bgpDSretryCount.decrementAndGet()) {
2109             try {
2110                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
2111                         InstanceIdentifier.create(Bgp.class)).orNull();
2112             } catch (ReadFailedException e) {
2113                 //Config DS may not be up, so sleep for 1 second and retry
2114                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
2115                 try {
2116                     Thread.sleep(WAIT_TIME_BETWEEN_EACH_TRY_MILLIS);
2117                 } catch (InterruptedException timerEx) {
2118                     LOG.debug("WAIT_TIME_BETWEEN_EACH_TRY_MILLIS, Timer got interrupted while waiting for"
2119                             + "config DS availability", timerEx);
2120                 }
2121             }
2122         }
2123         LOG.error("failed to get bgp config");
2124         return null;
2125     }
2126
2127     @SuppressWarnings("checkstyle:IllegalCatch")
2128     public synchronized boolean replay() {
2129         boolean replaySucceded = true;
2130         String host = getConfigHost();
2131         int port = getConfigPort();
2132         LOG.error("connecting  to bgp host {} ", host);
2133         boolean res = bgpRouter.connect(host, port);
2134         if (!res) {
2135             LOG.error("Cannot connect to BGP config server at {}:{}{}", host, port,
2136                     config != null ? "; Configuration Replay aborted" : "");
2137             return replaySucceded;
2138         }
2139         config = getConfig();
2140         if (config == null) {
2141             LOG.error("bgp config is empty nothing to push to bgp");
2142             return replaySucceded;
2143         }
2144         BgpRouter br = bgpRouter;
2145         AsId asId = config.getAsId();
2146         if (asId == null) {
2147             LOG.error("bgp as-id is null");
2148             return replaySucceded;
2149         }
2150         long asNum = asId.getLocalAs();
2151         IpAddress routerId = asId.getRouterId();
2152         String rid = routerId == null ? "" : routerId.stringValue();
2153         int stalepathTime = (int) getStalePathtime(RESTART_DEFAULT_GR, config.getAsId());
2154         boolean announceFbit = true;
2155         boolean replayDone = false;
2156         final int numberOfStartBgpRetries = 3;
2157         RetryOnException startBgpRetry = new RetryOnException(numberOfStartBgpRetries);
2158         do {
2159             try {
2160                 LOG.debug("Replaying BGPConfig ");
2161                 br.startBgp(asNum, rid, stalepathTime, announceFbit);
2162                 LOG.debug("Replay BGPConfig successful");
2163                 replayDone = true;
2164                 break;
2165             } catch (BgpRouterException bre) {
2166                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
2167                     LOG.debug("Starting the routesync for exception", bre);
2168                     startBgpRetry.errorOccured();
2169                     if (!startBgpRetry.shouldRetry()) {
2170                         LOG.debug("starting route sync for BgpRouter exception");
2171                         doRouteSync();
2172                     }
2173                 } else {
2174                     LOG.error("Replay: startBgp() received exception error {} : ",
2175                             bre.getErrorCode(), bre);
2176                     startBgpRetry.errorOccured();
2177                 }
2178             } catch (TApplicationException tae) {
2179                 if (tae.getType() == BgpRouterException.BGP_ERR_ACTIVE) {
2180                     LOG.debug("Starting the routesync for exception", tae);
2181                     startBgpRetry.errorOccured();
2182                     if (!startBgpRetry.shouldRetry()) {
2183                         LOG.debug("starting route sync for Thrift BGP_ERR_ACTIVE exception");
2184                         doRouteSync();
2185                     }
2186                 } else if (tae.getType() == BgpRouterException.BGP_ERR_COMMON_FAILURE) {
2187                     LOG.debug("Starting the routesync for AS-ID started exception", tae);
2188                     startBgpRetry.errorOccured();
2189                     if (!startBgpRetry.shouldRetry()) {
2190                         LOG.debug("starting route sync for Thrift BGP_ERR_COMMON_FAILURE exception");
2191                         doRouteSync();
2192                     }
2193                 } else {
2194                     LOG.error("Replay: startBgp() received exception type {}: ",
2195                             tae.getType(), tae);
2196                     startBgpRetry.errorOccured();
2197                 }
2198             } catch (Exception e) {
2199                 //not unusual. We may have restarted & BGP is already on
2200                 LOG.error("Replay:startBgp() received exception: ", e);
2201                 startBgpRetry.errorOccured();
2202             }
2203         } while (startBgpRetry.shouldRetry());
2204
2205         replaySucceded = replayDone;
2206
2207         startBgpCountersTask();
2208         startBgpAlarmsTask();
2209
2210         /*
2211          * commenting this due to a bug with QBGP. Will uncomment once QBGP fix is done.
2212          * This wont have any functional impacts
2213          */
2214         try {
2215             br.delayEOR(delayEorSeconds);
2216         } catch (TException | BgpRouterException e) {
2217             LOG.error("Replay: delayEOR() number of seconds to wait for EOR from ODL:", e);
2218         }
2219
2220         List<Neighbors> neighbors = config.getNeighbors();
2221         if (neighbors != null) {
2222             LOG.error("configuring existing Neighbors present for replay total neighbors {}", neighbors.size());
2223             boolean neighborConfigReplayResult = replayNbrConfig(neighbors, br);
2224             if (neighborConfigReplayResult == false) {
2225                 replaySucceded = false;
2226             }
2227         } else {
2228             LOG.error("no Neighbors present for replay config ");
2229         }
2230
2231         Logging logging = config.getLogging();
2232         if (logging != null) {
2233             try {
2234                 br.setLogging(logging.getFile(), logging.getLevel());
2235             } catch (TException | BgpRouterException e) {
2236                 LOG.error("Replay:setLogging() received exception", e);
2237             }
2238         }
2239
2240         GracefulRestart gracefulRestart = config.getGracefulRestart();
2241         if (gracefulRestart != null) {
2242             try {
2243                 br.addGracefulRestart(gracefulRestart.getStalepathTime().intValue());
2244             } catch (TException | BgpRouterException e) {
2245                 LOG.error("Replay:addGr() received exception", e);
2246             }
2247         }
2248
2249         List<Vrfs> vrfs = config.getVrfs();
2250         if (vrfs == null) {
2251             vrfs = new ArrayList<>();
2252         }
2253         for (Vrfs vrf : vrfs) {
2254             for (AddressFamiliesVrf adf : vrf.getAddressFamiliesVrf()) {
2255                 try {
2256                     br.addVrf(BgpUtil.getLayerType(adf), vrf.getRd(), vrf.getImportRts(),
2257                             vrf.getExportRts());
2258                 } catch (TException | BgpRouterException e) {
2259                     LOG.error("Replay:addVrf() received exception", e);
2260                 }
2261             }
2262         }
2263
2264         List<Networks> ln = config.getNetworks();
2265         if (ln != null) {
2266             for (Networks net : ln) {
2267                 String rd = net.getRd();
2268                 String pfxlen = net.getPrefixLen();
2269                 String nh = net.getNexthop().getValue();
2270                 Long label = net.getLabel();
2271                 int lbl = label == null ? 0 : label.intValue();
2272                 int l3vni = net.getL3vni() == null ? 0 : net.getL3vni().intValue();
2273                 int l2vni = net.getL2vni() == null ? 0 : net.getL2vni().intValue();
2274                 if (rd == null && lbl > 0) {
2275                     //LU prefix is being deleted.
2276                     rd = Integer.toString(lbl);
2277                 }
2278
2279                 BgpControlPlaneType protocolType = net.getBgpControlPlaneType();
2280                 int ethernetTag = net.getEthtag().intValue();
2281                 String esi = net.getEsi();
2282                 String macaddress = net.getMacaddress();
2283                 EncapType encapType = net.getEncapType();
2284                 String routerMac = net.getRoutermac();
2285
2286                 try {
2287                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni,
2288                             BgpUtil.convertToThriftProtocolType(protocolType),
2289                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
2290                 } catch (TException | BgpRouterException e) {
2291                     LOG.error("Replay:addPfx() received exception", e);
2292                 }
2293             }
2294         }
2295
2296
2297         List<Multipath> multipaths = config.getMultipath();
2298
2299         if (multipaths != null) {
2300             for (Multipath multipath : multipaths) {
2301                 if (multipath != null) {
2302                     af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
2303                     af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
2304
2305                     try {
2306                         if (multipath.isMultipathEnabled()) {
2307                             br.enableMultipath(afi, safi);
2308                         } else {
2309                             br.disableMultipath(afi, safi);
2310                         }
2311                     } catch (TException | BgpRouterException e) {
2312                         LOG.info("Replay:multipaths() received exception", e);
2313                     }
2314                 }
2315             }
2316         }
2317         List<VrfMaxpath> vrfMaxpaths = config.getVrfMaxpath();
2318         if (vrfMaxpaths != null) {
2319             for (VrfMaxpath vrfMaxpath : vrfMaxpaths) {
2320                 try {
2321                     br.multipaths(vrfMaxpath.getRd(), vrfMaxpath.getMaxpaths());
2322                 } catch (TException | BgpRouterException e) {
2323                     LOG.info("Replay:vrfMaxPath() received exception", e);
2324                 }
2325             }
2326         }
2327
2328         //send End of Rib Marker to Qthriftd.
2329         final int numberOfEORRetries = 3;
2330         replayDone = false;
2331         RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
2332         do {
2333             try {
2334                 br.sendEOR();
2335                 LOG.debug("Replay sendEOR {} successful");
2336                 replayDone = true;
2337                 break;
2338             } catch (Exception e) {
2339                 eorRetry.errorOccured();
2340                 LOG.error("Replay:sedEOR() received exception:", e);
2341             }
2342         } while (eorRetry.shouldRetry());
2343
2344         return replaySucceded && replayDone;
2345     }
2346
2347     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
2348         bgpUtil.update(iid, dto);
2349     }
2350
2351     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
2352         bgpUtil.delete(iid);
2353     }
2354
2355     public void startConfig(String bgpHost, int thriftPort) {
2356         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2357                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2358         InstanceIdentifier<ConfigServer> iid = iib.build();
2359         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
2360         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
2361                 .setPort((long) thriftPort).build();
2362         update(iid, dto);
2363     }
2364
2365     public void startBgp(long as, String routerId, int spt, boolean fbit) {
2366         IpAddress rid = routerId == null ? null : IpAddressBuilder.getDefaultInstance(routerId);
2367         Long staleTime = (long) spt;
2368         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2369                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2370         InstanceIdentifier<AsId> iid = iib.build();
2371         AsId dto = new AsIdBuilder().setLocalAs(as)
2372                 .setRouterId(rid)
2373                 .setStalepathTime(staleTime)
2374                 .setAnnounceFbit(fbit).build();
2375         update(iid, dto);
2376     }
2377
2378     public void addLogging(String fileName, String logLevel) {
2379         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2380                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2381         InstanceIdentifier<Logging> iid = iib.build();
2382         Logging dto = new LoggingBuilder().setFile(fileName)
2383                 .setLevel(logLevel).build();
2384         update(iid, dto);
2385     }
2386
2387     public void addGracefulRestart(int staleTime) {
2388         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2389                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
2390         InstanceIdentifier<GracefulRestart> iid = iib.build();
2391         GracefulRestart dto = new GracefulRestartBuilder()
2392                 .setStalepathTime((long) staleTime).build();
2393         update(iid, dto);
2394     }
2395
2396     public void addNeighbor(
2397             String nbrIp, long remoteAs, @Nullable final TcpMd5SignaturePasswordType md5Secret) {
2398         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2399         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2400                 InstanceIdentifier.builder(Bgp.class)
2401                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2402         InstanceIdentifier<Neighbors> iid = iib.build();
2403         TcpSecurityOption tcpSecOption = null;
2404         if (md5Secret != null) {
2405             tcpSecOption = new TcpMd5SignatureOptionBuilder().setTcpMd5SignaturePassword(md5Secret).build();
2406         } // else let tcpSecOption be null
2407         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
2408                 .setRemoteAs(remoteAs).setTcpSecurityOption(tcpSecOption).build();
2409         update(iid, dto);
2410     } // public addNeighbor(nbrIp, remoteAs, md5Secret)
2411
2412     public void addUpdateSource(String nbrIp, String srcIp) {
2413         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2414         Ipv4Address srcAddr = new Ipv4Address(srcIp);
2415         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2416                 InstanceIdentifier.builder(Bgp.class)
2417                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2418                         .child(UpdateSource.class);
2419         InstanceIdentifier<UpdateSource> iid = iib.build();
2420         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
2421                 .setSourceIp(srcAddr).build();
2422         update(iid, dto);
2423     }
2424
2425     public void addEbgpMultihop(String nbrIp, int hops) {
2426         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2427         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2428                 InstanceIdentifier.builder(Bgp.class)
2429                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2430                         .child(EbgpMultihop.class);
2431         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2432         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
2433                 .setNhops((long) hops).build();
2434         update(iid, dto);
2435     }
2436
2437     public void addAddressFamily(String nbrIp, int afi, int safi) {
2438         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2439         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2440                 InstanceIdentifier.builder(Bgp.class)
2441                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2442                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2443         InstanceIdentifier<AddressFamilies> iid = iib.build();
2444         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
2445                 .setAfi((long) afi).setSafi((long) safi).build();
2446         update(iid, dto);
2447     }
2448
2449     public void addPrefix(String rd, String macAddress, String pfx, List<String> nhList,
2450               VrfEntry.EncapType encapType, long lbl, long l3vni, long l2vni, String gatewayMac) {
2451         for (String nh : nhList) {
2452             Ipv4Address nexthop = nh != null ? new Ipv4Address(nh) : null;
2453             Long label = lbl;
2454             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
2455                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
2456             NetworksBuilder networksBuilder = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
2457                                                 .setLabel(label).setEthtag(BgpConstants.DEFAULT_ETH_TAG);
2458             buildVpnEncapSpecificInfo(networksBuilder, encapType, label, l3vni, l2vni, macAddress, gatewayMac);
2459             update(iid, networksBuilder.build());
2460         }
2461     }
2462
2463     private static void buildVpnEncapSpecificInfo(NetworksBuilder builder, VrfEntry.EncapType encapType, long label,
2464                                                   long l3vni, long l2vni, String macAddress, String gatewayMac) {
2465         if (encapType.equals(VrfEntry.EncapType.Mplsgre)) {
2466             builder.setLabel(label).setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLL3VPN)
2467                     .setEncapType(EncapType.GRE);
2468         } else {
2469             builder.setL3vni(l3vni).setL2vni(l2vni).setMacaddress(macAddress).setRoutermac(gatewayMac)
2470                     .setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLEVPN).setEncapType(EncapType.VXLAN);
2471         }
2472     }
2473
2474     // TODO: add LayerType as arg - supports command
2475     public void addVrf(String rd, List<String> irts, List<String> erts, AddressFamily addressFamily) {
2476         Vrfs vrf = bgpUtil.getVrfFromRd(rd);
2477         List<AddressFamiliesVrf> adfList = new ArrayList<>(1);
2478         if (vrf != null) {
2479             adfList = vrf.getAddressFamiliesVrf();
2480         }
2481         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2482         if (addressFamily.equals(AddressFamily.IPV4)) {
2483             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2484             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2485         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2486             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2487             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2488         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2489             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2490             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2491         }
2492         AddressFamiliesVrf adf = adfBuilder.build();
2493         adfList.add(adf);
2494         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib = InstanceIdentifier.builder(Bgp.class)
2495                 .child(Vrfs.class, new VrfsKey(rd));
2496         InstanceIdentifier<Vrfs> iid = iib.build();
2497         Vrfs dto = new VrfsBuilder().setRd(rd).setImportRts(irts)
2498             .setExportRts(erts).setAddressFamiliesVrf(adfList).build();
2499
2500         List<AddressFamiliesVrf> listAdFamilies = mapNewAdFamily.get(rd);
2501         if (listAdFamilies != null) {
2502             listAdFamilies.add(adf);
2503         } else {
2504             mapNewAdFamily.put(rd, adfList);
2505         }
2506
2507         try {
2508             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
2509         } catch (TransactionCommitFailedException e) {
2510             LOG.error("Error adding VRF to datastore", e);
2511             throw new RuntimeException(e);
2512         }
2513     }
2514
2515     public void stopConfig() {
2516         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2517                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2518         InstanceIdentifier<ConfigServer> iid = iib.build();
2519         delete(iid);
2520     }
2521
2522     public void stopBgp() {
2523         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2524                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2525         InstanceIdentifier<AsId> iid = iib.build();
2526         delete(iid);
2527     }
2528
2529     public void delLogging() {
2530         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2531                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2532         InstanceIdentifier<Logging> iid = iib.build();
2533         delete(iid);
2534     }
2535
2536     public void delGracefulRestart() {
2537         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2538                 InstanceIdentifier.builder(Bgp.class)
2539                         .child(GracefulRestart.class);
2540         InstanceIdentifier<GracefulRestart> iid = iib.build();
2541         delete(iid);
2542     }
2543
2544     public void delNeighbor(String nbrIp) {
2545         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2546         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2547                 InstanceIdentifier.builder(Bgp.class)
2548                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2549         InstanceIdentifier<Neighbors> iid = iib.build();
2550         delete(iid);
2551     }
2552
2553     public void delUpdateSource(String nbrIp) {
2554         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2555         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2556                 InstanceIdentifier.builder(Bgp.class)
2557                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2558                         .child(UpdateSource.class);
2559         InstanceIdentifier<UpdateSource> iid = iib.build();
2560         delete(iid);
2561     }
2562
2563     public void delEbgpMultihop(String nbrIp) {
2564         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2565         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2566                 InstanceIdentifier.builder(Bgp.class)
2567                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2568                         .child(EbgpMultihop.class);
2569         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2570         delete(iid);
2571     }
2572
2573     public void delAddressFamily(String nbrIp, int afi, int safi) {
2574         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2575         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2576                 InstanceIdentifier.builder(Bgp.class)
2577                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2578                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2579         InstanceIdentifier<AddressFamilies> iid = iib.build();
2580         delete(iid);
2581     }
2582
2583     public void delPrefix(String rd, String pfx) {
2584         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
2585                 InstanceIdentifier.builder(Bgp.class)
2586                         .child(Networks.class, new NetworksKey(pfx, rd));
2587         InstanceIdentifier<Networks> iid = iib.build();
2588         delete(iid);
2589     }
2590
2591     public boolean delVrf(String rd, AddressFamily addressFamily) {
2592         if (addressFamily == null) {
2593             LOG.error("delVrf: vrf {}, addressFamily invalid", rd);
2594             return false;
2595         }
2596         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2597         if (addressFamily.equals(AddressFamily.IPV4)) {
2598             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2599             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2600         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2601             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2602             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2603         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2604             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2605             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2606         }
2607         Vrfs vrfOriginal = bgpUtil.getVrfFromRd(rd);
2608         if (vrfOriginal == null) {
2609             LOG.error("delVrf: no vrf with existing rd {}. step aborted", rd);
2610             return false;
2611         }
2612
2613         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2614                 InstanceIdentifier.builder(Bgp.class)
2615                         .child(Vrfs.class, new VrfsKey(rd));
2616
2617         InstanceIdentifier<Vrfs> iid = iib.build();
2618
2619         InstanceIdentifier.builder(Bgp.class).build()
2620                 .child(Multipath.class, new MultipathKey(adfBuilder.getAfi(), adfBuilder.getSafi()));
2621         @SuppressWarnings("static-access")
2622         InstanceIdentifier<Bgp> iid6 =  InstanceIdentifier.create(Bgp.class);
2623         InstanceIdentifierBuilder<Vrfs> iib3 = iid6.child(Vrfs.class, new VrfsKey(rd)).builder();
2624         InstanceIdentifier<Vrfs> iidFinal = iib3.build();
2625
2626         //** update or delete the vrfs with the rest of AddressFamilies already present in the last list
2627         AddressFamiliesVrf adfToDel = adfBuilder.build();
2628         List<AddressFamiliesVrf> adfListOriginal = vrfOriginal.getAddressFamiliesVrf() == null
2629                 ? new ArrayList<>() : vrfOriginal.getAddressFamiliesVrf();
2630         List<AddressFamiliesVrf> adfListToRemoveFromOriginal = new ArrayList<>();
2631         adfListOriginal.forEach(adf -> {
2632             if (adf.equals(adfToDel)) {
2633                 adfListToRemoveFromOriginal.add(adfToDel);
2634                 return;
2635             }
2636         });
2637         for (AddressFamiliesVrf adfToRemove : adfListToRemoveFromOriginal) {
2638             adfListOriginal.remove(adfToRemove);
2639             try {
2640                 SingleTransactionDataBroker.syncWrite(dataBroker,
2641                         LogicalDatastoreType.CONFIGURATION, iid, vrfOriginal);
2642             } catch (TransactionCommitFailedException e) {
2643                 LOG.error("delVrf: Error updating VRF to datastore", e);
2644                 throw new RuntimeException(e);
2645             }
2646         }
2647         if (adfListOriginal.isEmpty()) {
2648             delete(iidFinal);
2649             return true;
2650         }
2651         // not all is removed
2652         return false;
2653     }
2654
2655     public void setMultipathStatus(af_afi afi, af_safi safi, boolean enable) {
2656         long lafi = afi.getValue();
2657         long lsafi = safi.getValue();
2658
2659         InstanceIdentifier.InstanceIdentifierBuilder<Multipath> iib =
2660                 InstanceIdentifier
2661                         .builder(Bgp.class)
2662                         .child(Multipath.class,
2663                                 new MultipathKey(Long.valueOf(afi.getValue()), Long.valueOf(safi.getValue())));
2664
2665         Multipath dto = new MultipathBuilder().setAfi(lafi).setSafi(lsafi).setMultipathEnabled(enable).build();
2666         update(iib.build(), dto);
2667     }
2668
2669     public void multipaths(String rd, int maxpath) {
2670         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
2671                 InstanceIdentifier
2672                         .builder(Bgp.class)
2673                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
2674
2675         VrfMaxpath dto = new VrfMaxpathBuilder().setRd(rd).setMaxpaths(maxpath).build();
2676         update(iib.build(), dto);
2677     }
2678
2679     /*
2680     * Remove Stale Marked Routes after timer expiry.
2681     */
2682     private class RouteCleanup implements Callable<Integer> {
2683
2684         @Override
2685         public Integer call() {
2686             totalCleared = 0;
2687             try {
2688                 if (staledFibEntriesMap.isEmpty()) {
2689                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
2690                 } else {
2691                     for (String rd : staledFibEntriesMap.keySet()) {
2692                         if (Thread.interrupted()) {
2693                             return 0;
2694                         }
2695                         Map<String, Long> map = staledFibEntriesMap.get(rd);
2696                         if (map != null) {
2697                             for (String key : map.keySet()) {
2698                                 if (Thread.interrupted()) {
2699                                     return 0;
2700                                 }
2701                                 String prefix = extractPrefix(key);
2702                                 String nextHop = extractNextHop(key);
2703                                 totalCleared++;
2704                                 LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}, nextHop:{}",
2705                                         rd, prefix, nextHop);
2706                                 fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix, nextHop);
2707                             }
2708                         }
2709                     }
2710                 }
2711             } finally {
2712                 staledFibEntriesMap.clear();
2713             }
2714             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
2715             return 0;
2716         }
2717     }
2718
2719     /*
2720      * BGP restart scenario, ODL-BGP manager was/is running.
2721      * On re-sync notification, Get a copy of FIB database.
2722      */
2723     public void createStaleFibMap() {
2724         totalStaledCount = 0;
2725         try {
2726             staledFibEntriesMap.clear();
2727             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2728
2729             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2730                     LogicalDatastoreType.CONFIGURATION, id);
2731             if (fibEntries.isPresent()) {
2732                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2733                 for (VrfTables vrfTable : staleVrfTables) {
2734                     Map<String, Long> staleFibEntMap = new HashMap<>();
2735                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2736                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2737                             //Stale marking and cleanup is only meant for the routes learned through BGP.
2738                             continue;
2739                         }
2740                         if (Thread.interrupted()) {
2741                             break;
2742                         }
2743                         totalStaledCount++;
2744                         //Create MAP from staleVrfTables.
2745                         vrfEntry.getRoutePaths()
2746                                 .forEach(
2747                                     routePath -> staleFibEntMap.put(
2748                                             appendNextHopToPrefix(vrfEntry.getDestPrefix(),
2749                                                     routePath.getNexthopAddress()), routePath.getLabel()));
2750                     }
2751                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), staleFibEntMap);
2752                 }
2753             } else {
2754                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
2755             }
2756         } catch (ReadFailedException e) {
2757             LOG.error("createStaleFibMap:: error ", e);
2758         }
2759         LOG.error("created {} staled entries ", totalStaledCount);
2760     }
2761
2762     /*
2763      * BGP config remove scenario, Need to remove all the
2764      * external routes from FIB.
2765      */
2766     public void deleteExternalFibRoutes() {
2767         totalExternalRoutes = 0;
2768         totalExternalMacRoutes = 0;
2769         try {
2770             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2771
2772             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2773                     LogicalDatastoreType.CONFIGURATION, id);
2774             if (fibEntries.isPresent()) {
2775                 if (fibEntries.get().getVrfTables() == null) {
2776                     LOG.error("deleteExternalFibRoutes::getVrfTables is null");
2777                     return;
2778                 }
2779                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2780                 for (VrfTables vrfTable : staleVrfTables) {
2781                     String rd = vrfTable.getRouteDistinguisher();
2782                     if (vrfTable.getVrfEntry() != null) {
2783                         for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2784                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2785                                 //route cleanup is only meant for the routes learned through BGP.
2786                                 continue;
2787                             }
2788                             totalExternalRoutes++;
2789                             fibDSWriter.removeFibEntryFromDS(rd, vrfEntry.getDestPrefix());
2790                         }
2791                     } else if (vrfTable.getMacVrfEntry() != null) {
2792                         for (MacVrfEntry macEntry : vrfTable.getMacVrfEntry()) {
2793                             if (RouteOrigin.value(macEntry.getOrigin()) != RouteOrigin.BGP) {
2794                                 //route cleanup is only meant for the routes learned through BGP.
2795                                 continue;
2796                             }
2797                             totalExternalMacRoutes++;
2798                             fibDSWriter.removeMacEntryFromDS(rd, macEntry.getMac());
2799                         }
2800                     }
2801                 }
2802             } else {
2803                 LOG.error("deleteExternalFibRoutes:: FIBentries.class is not present");
2804             }
2805         } catch (ReadFailedException e) {
2806             LOG.error("deleteExternalFibRoutes:: error ", e);
2807         }
2808         LOG.debug("deleted {} fib entries {} mac entries", totalExternalRoutes, totalExternalMacRoutes);
2809     }
2810
2811     public boolean addToRt2TepMap(String rd, String tepIp, String mac, Long l2vni) {
2812         boolean isFirstMacUpdateFromTep = false;
2813         if (rt2TepMap.containsKey(rd)) {
2814             if (rt2TepMap.get(rd).containsKey(tepIp)) {
2815                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and tep-ip {}. No Elan DS write required",
2816                         mac, l2vni, rd, tepIp);
2817                 rt2TepMap.get(rd).get(tepIp).put(mac, l2vni);
2818             } else {
2819                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and new tep-ip {}",
2820                         mac, l2vni, rd, tepIp);
2821                 isFirstMacUpdateFromTep = true;
2822                 Map<String, Long> macList = new HashMap<>();
2823                 macList.put(mac, l2vni);
2824                 rt2TepMap.get(rd).put(tepIp, macList);
2825             }
2826         } else {
2827             LOG.debug("RT2 with mac {} l2vni {} from new rd {} and tep ip {}",
2828                     mac, l2vni, rd, tepIp);
2829             isFirstMacUpdateFromTep = true;
2830             Map<String, Long> macList = new HashMap<>();
2831             macList.put(mac, l2vni);
2832             Map<String, Map<String, Long>> tepIpMacMap = new HashMap<>();
2833             tepIpMacMap.put(tepIp, macList);
2834             rt2TepMap.put(rd, tepIpMacMap);
2835         }
2836         return isFirstMacUpdateFromTep;
2837     }
2838
2839     public boolean deleteFromRt2TepMap(String rd, String tepIp, String mac) {
2840         boolean isLastMacUpdateFromTep = false;
2841         LOG.debug("RT2 withdraw with rd {} mac {} tep-ip {} ", rd, mac, tepIp);
2842         if (rt2TepMap.containsKey(rd)) {
2843             if (rt2TepMap.get(rd).containsKey(tepIp)) {
2844                 if (rt2TepMap.get(rd).get(tepIp).containsKey(mac)) {
2845                     LOG.debug("RT2 Withdraw : Removing the mac {} from Map", mac);
2846                     rt2TepMap.get(rd).get(tepIp).remove(mac);
2847                     if (rt2TepMap.get(rd).get(tepIp).isEmpty()) {
2848                         isLastMacUpdateFromTep = true;
2849                         LOG.debug("RT2 Withdraw : Removing the tep-ip {} from Map", tepIp);
2850                         rt2TepMap.get(rd).remove(tepIp);
2851                         if (rt2TepMap.get(rd).isEmpty()) {
2852                             LOG.debug("RT2 Withdraw : Removing the rd {} from Map", rd);
2853                             rt2TepMap.remove(rd);
2854                         }
2855                     }
2856                 }
2857             }
2858         }
2859         return isLastMacUpdateFromTep;
2860     }
2861
2862     public Collection<String> getTepIPs(String rd) {
2863         final Map<String, Map<String, Long>> tepIpMap = rt2TepMap.get(rd);
2864         return tepIpMap != null ? tepIpMap.keySet() : Collections.emptyList();
2865     }
2866
2867     public boolean isBgpConnected() {
2868         return bgpRouter.isBgpConnected();
2869     }
2870
2871     public long getLastConnectedTS() {
2872         return bgpRouter.getLastConnectedTS();
2873     }
2874
2875     public long getConnectTS() {
2876         return bgpRouter.getConnectTS();
2877     }
2878
2879     public long getStartTS() {
2880         return bgpRouter.getStartTS();
2881     }
2882
2883     public TTransport getTransport() {
2884         return bgpRouter.getTransport();
2885     }
2886
2887     public int getTotalStaledCount() {
2888         return totalStaledCount;
2889     }
2890
2891     public int getTotalCleared() {
2892         return totalCleared;
2893     }
2894
2895     public BgpCounters getBgpCounters() {
2896         return bgpCountersReference.get();
2897     }
2898
2899     private void startBgpCountersTask() {
2900         if (getBgpCounters() == null && bgpCountersReference.compareAndSet(null,
2901                 new BgpCounters(getBgpSdncMipIp(), metricProvider))) {
2902             bgpCountersTask = executor.scheduleAtFixedRate(bgpCountersReference.get(), 0, 120 * 1000,
2903                     TimeUnit.MILLISECONDS);
2904             LOG.info("Bgp Counters task scheduled for every two minutes.");
2905
2906             addLogging(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
2907         }
2908     }
2909
2910     private void stopBgpCountersTask() {
2911         final BgpCounters bgpCounters = bgpCountersReference.getAndSet(null);
2912         if (bgpCounters != null) {
2913             bgpCountersTask.cancel(true);
2914             bgpCounters.close();
2915         }
2916     }
2917
2918     private void startBgpAlarmsTask() {
2919         if (getBgpAlarms() == null && bgpAlarmsReference.compareAndSet(null, new BgpAlarms(this))) {
2920             bgpAlarmsReference.get().init();
2921             bgpAlarmsTask = executor.scheduleAtFixedRate(bgpAlarmsReference.get(), 0, 60 * 1000, TimeUnit.MILLISECONDS);
2922             LOG.info("Bgp Alarms task scheduled for every minute.");
2923         }
2924     }
2925
2926     private void stopBgpAlarmsTask() {
2927         final BgpAlarms bgpAlarms = bgpAlarmsReference.getAndSet(null);
2928         if (bgpAlarms != null) {
2929             bgpAlarmsTask.cancel(true);
2930             bgpAlarms.close();
2931         }
2932     }
2933
2934     public BgpAlarms getBgpAlarms() {
2935         return bgpAlarmsReference.get();
2936     }
2937
2938     private static String appendNextHopToPrefix(String prefix, String nextHop) {
2939         return prefix + ":" + nextHop;
2940     }
2941
2942     private static String extractPrefix(String prefixNextHop) {
2943         return prefixNextHop.split(":")[0];
2944     }
2945
2946     private static String extractNextHop(String prefixNextHop) {
2947         return prefixNextHop.split(":")[1];
2948     }
2949
2950     private static String extractMd5Secret(final Neighbors val) {
2951         String md5Secret = null;
2952         TcpSecurityOption tcpSecOpt = val.getTcpSecurityOption();
2953         if (tcpSecOpt != null) {
2954             if (tcpSecOpt instanceof TcpMd5SignatureOption) {
2955                 md5Secret = ((TcpMd5SignatureOption) tcpSecOpt).getTcpMd5SignaturePassword().getValue();
2956             } else { // unknown TcpSecurityOption
2957                 LOG.debug("neighbors  Ignored unknown tcp-security-option of peer {}", val.getAddress().getValue());
2958             }
2959         }
2960         return md5Secret;
2961     } // private method extractMd5Secret
2962 }