bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import static org.opendaylight.netvirt.bgpmanager.oam.BgpConstants.HISTORY_LIMIT;
  11 import static org.opendaylight.netvirt.bgpmanager.oam.BgpConstants.HISTORY_THRESHOLD;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.base.Preconditions;
  15 import com.google.common.net.InetAddresses;
  16 import com.google.common.util.concurrent.Futures;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18 import com.google.common.util.concurrent.ThreadFactoryBuilder;
  19 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  20 import io.netty.util.concurrent.GlobalEventExecutor;
  21 import java.lang.reflect.Constructor;
  22 import java.lang.reflect.InvocationTargetException;
  23 import java.net.InetAddress;
  24 import java.net.InetSocketAddress;
  25 import java.net.NetworkInterface;
  26 import java.net.SocketException;
  27 import java.util.ArrayList;
  28 import java.util.Collection;
  29 import java.util.Collections;
  30 import java.util.Enumeration;
  31 import java.util.HashMap;
  32 import java.util.Iterator;
  33 import java.util.List;
  34 import java.util.Map;
  35 import java.util.Objects;
  36 import java.util.Timer;
  37 import java.util.concurrent.Callable;
  38 import java.util.concurrent.ConcurrentHashMap;
  39 import java.util.concurrent.CountDownLatch;
  40 import java.util.concurrent.ExecutionException;
  41 import java.util.concurrent.Executors;
  42 import java.util.concurrent.Future;
  43 import java.util.concurrent.ScheduledExecutorService;
  44 import java.util.concurrent.ScheduledFuture;
  45 import java.util.concurrent.TimeUnit;
  46 import java.util.concurrent.TimeoutException;
  47 import java.util.concurrent.atomic.AtomicBoolean;
  48 import java.util.concurrent.atomic.AtomicInteger;
  49 import java.util.concurrent.atomic.AtomicReference;
  50 import javax.annotation.PreDestroy;
  51 import javax.inject.Inject;
  52 import javax.inject.Singleton;
  53 import org.apache.thrift.TApplicationException;
  54 import org.apache.thrift.TException;
  55 import org.apache.thrift.transport.TTransport;
  56 import org.eclipse.jdt.annotation.Nullable;
  57 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  58 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  59 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  60 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  61 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  62 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  63 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  64 import org.opendaylight.genius.mdsalutil.NwConstants;
  65 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  66 import org.opendaylight.infrautils.metrics.MetricProvider;
  67 import org.opendaylight.mdsal.eos.binding.api.Entity;
  68 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipCandidateRegistration;
  69 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipListenerRegistration;
  70 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  71 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  72 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  73 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  74 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  75 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  76 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  77 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  78 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  79 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  80 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  81 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  82 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  83 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  84 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  85 import org.opendaylight.netvirt.bgpmanager.thrift.gen.protocol_type;
  86 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  87 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  88 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  89 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  90 import org.opendaylight.ovsdb.utils.mdsal.utils.TransactionHistory;
  91 import org.opendaylight.ovsdb.utils.mdsal.utils.TransactionType;
  92 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebfd.rev190219.BfdConfig;
  93 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebfd.rev190219.BfdConfigBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.AddressFamily;
  95 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  96 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
  97 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EbgpService;
  98 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EncapType;
  99 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorInput;
 100 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorOutput;
 101 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorOutputBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.TcpMd5SignaturePasswordType;
 103 //import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.*;
 104 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
 105 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
 107 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.DcgwTepList;
 109 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
 110 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
 112 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathContainer;
 114 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsContainer;
 115 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksContainer;
 116 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathContainer;
 117 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsContainer;
 118 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTep;
 119 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTepBuilder;
 120 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTepKey;
 121 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.Multipath;
 122 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.MultipathBuilder;
 123 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.MultipathKey;
 124 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.Neighbors;
 125 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.NeighborsBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.NeighborsKey;
 127 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamilies;
 128 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamiliesBuilder;
 129 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamiliesKey;
 130 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.EbgpMultihop;
 131 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.EbgpMultihopBuilder;
 132 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.UpdateSource;
 133 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.UpdateSourceBuilder;
 134 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.Networks;
 135 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.NetworksBuilder;
 136 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.NetworksKey;
 137 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpath;
 138 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpathBuilder;
 139 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpathKey;
 140 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.Vrfs;
 141 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.VrfsBuilder;
 142 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.VrfsKey;
 143 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.vrfs.AddressFamiliesVrf;
 144 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.vrfs.AddressFamiliesVrfBuilder;
 145 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.TcpSecurityOption;
 146 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOption;
 147 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOptionBuilder;
 148 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
 149 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
 150 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
 151 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 152 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 153 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.macvrfentries.MacVrfEntry;
 154 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 155 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 156 import org.opendaylight.yangtools.yang.binding.DataObject;
 157 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 158 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 159 import org.opendaylight.yangtools.yang.common.RpcError;
 160 import org.opendaylight.yangtools.yang.common.RpcResult;
 161 import org.opendaylight.yangtools.yang.common.RpcResultBuilder;
 162 import org.opendaylight.yangtools.yang.common.Uint32;
 163 import org.osgi.framework.BundleContext;
 164 import org.osgi.util.tracker.ServiceTracker;
 165 import org.slf4j.Logger;
 166 import org.slf4j.LoggerFactory;
 167
 168 @Singleton
 169 public class BgpConfigurationManager implements EbgpService {
 170     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 171
 172     // to have stale FIB map (RD, Prefix)
 173     //  number of seconds wait for route sync-up between ODL and BGP
 174     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 175     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 176     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 177     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 178     private static final String DEF_UPORT = "6644";
 179     private static final String DEF_CHOST = "255.255.255.255"; // Invalid Host IP
 180     private static final String DEF_CPORT = "7644";
 181     private static final String DEF_BGP_SDNC_MIP = "127.0.0.1";
 182     //vpnservice.bgp.thrift.bgp.mip is the MIP present with ODL. Here we open 6644 port
 183     private static final String BGP_SDNC_MIP = "vpnservice.bgp.thrift.bgp.mip";
 184     private static final String BGP_GR_RESTART_TIMER_PROPERTY = "vpnservice.bgp.gr.timer";
 185     private static final String BGP_KA_TIMER_PROPERTY = "vpnservice.bgp.ka.timer";
 186     private static final String BGP_HOLD_TIMER_PROPERTY = "vpnservice.bgp.hold.timer";
 187     private static final String BGP_EOR_DELAY_PROPERTY = "vpnservice.bgp.eordelay";
 188     private static final int DEF_BGP_KA_TIME = 60;
 189     private static final int DEF_BGP_HOLD_TIME = 180;
 190     private static final int DEF_BGP_GR_TIME = 4000;
 191     private static final int RESTART_DEFAULT_GR = 90;
 192     private static final int DS_RETRY_COUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
 193     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
 194     private static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 195     private static final String BGP_EOR_DELAY = "vpnservice.bgp.eordelay";
 196     private static final String DEF_BGP_EOR_DELAY = "1800";
 197     private static final String BGP_ENTITY_NAME = "bgp";
 198     private static final String ADD_WARN = "Config store updated; undo with Delete if needed.";
 199     private static final String DEL_WARN = "Config store updated; undo with Add if needed.";
 200     private static final String UPD_WARN = "Update operation not supported; Config store updated;"
 201             + " restore with another Update if needed.";
 202     private static long bgp_as_num = 0;
 203     private int bgpKaTime = 0;
 204     private int bgpHoldTime = 0;
 205     private int bgpGrRestartTime = 0;
 206
 207     private static final Class<?>[] REACTORS = {
 208         ConfigServerReactor.class, AsIdReactor.class,
 209         GracefulRestartReactor.class, LoggingReactor.class,
 210         NeighborsReactor.class, UpdateSourceReactor.class,
 211         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 212         NetworksReactor.class, VrfsReactor.class, BgpReactor.class,
 213         MultipathReactor.class, VrfMaxpathReactor.class, BfdConfigReactor.class
 214     };
 215
 216     private IBgpManager bgpManager;
 217     private final DataBroker dataBroker;
 218     private final FibDSWriter fibDSWriter;
 219     private final IVpnLinkService vpnLinkService;
 220     private final BundleContext bundleContext;
 221     private final BgpUtil bgpUtil;
 222     private volatile Bgp config;
 223     private final BgpRouter bgpRouter;
 224     private final BgpSyncHandle bgpSyncHandle = new BgpSyncHandle();
 225     private volatile BgpThriftService bgpThriftService = null;
 226     private final int delayEorSeconds;
 227
 228     private final CountDownLatch initer = new CountDownLatch(1);
 229
 230     private final String hostStartup;
 231     private final String portStartup;
 232
 233     private final AtomicReference<BgpCounters> bgpCountersReference = new AtomicReference<>();
 234     private ScheduledFuture<?> bgpCountersTask;
 235
 236     private final AtomicReference<BgpAlarms> bgpAlarmsReference = new AtomicReference<>();
 237     private ScheduledFuture<?> bgpAlarmsTask;
 238
 239     private Future<?> lastReplayJobFt;
 240     private ScheduledFuture<?> routeCleanupFuture;
 241
 242     private long staleStartTime;
 243     private long staleEndTime;
 244     private long cfgReplayStartTime;
 245     private long cfgReplayEndTime;
 246     private long staleCleanupTime;
 247     private int totalStaledCount;
 248     private int totalCleared;
 249     private int totalExternalRoutes;
 250     private int totalExternalMacRoutes;
 251
 252     private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(
 253             new ThreadFactoryBuilder().setNameFormat("bgp-config-%d").setDaemon(true).build());
 254
 255     /**
 256      * this map store the new address families to send to quagga. When it is sended you must clear it.
 257      * The keys String are rd (route distinguisher).
 258      * */
 259     private final ConcurrentHashMap<String, List<AddressFamiliesVrf>> mapNewAdFamily = new ConcurrentHashMap<>();
 260
 261     // map<rd, map<prefix/len:nexthop, label>>
 262     private final Map<String, Map<String, Uint32>> staledFibEntriesMap = new ConcurrentHashMap<>();
 263
 264     // map<rd, map<tep-ip, map<mac, l2vni>>>
 265     private final Map<String, Map<String, Map<String, Uint32>>> rt2TepMap = new ConcurrentHashMap<>();
 266
 267     private final List<AutoCloseable> listeners = new ArrayList<>();
 268
 269     private final EntityOwnershipUtils entityOwnershipUtils;
 270     private final EntityOwnershipCandidateRegistration candidateRegistration;
 271     private final EntityOwnershipListenerRegistration entityListenerRegistration;
 272     private final MetricProvider metricProvider;
 273     private final TransactionHistory bgpUpdatesHistory;
 274
 275     private volatile AtomicBoolean eorSupressedDuetoUpgradeFlag = new AtomicBoolean(false);
 276
 277     @Inject
 278     public BgpConfigurationManager(final DataBroker dataBroker,
 279             final EntityOwnershipService entityOwnershipService,
 280             final FibDSWriter fibDSWriter,
 281             final IVpnLinkService vpnLinkSrvce,
 282             final BundleContext bundleContext,
 283             final BgpUtil bgpUtil,
 284             final MetricProvider metricProvider) {
 285         this.dataBroker = dataBroker;
 286         this.fibDSWriter = fibDSWriter;
 287         this.vpnLinkService = vpnLinkSrvce;
 288         this.bundleContext = bundleContext;
 289         this.bgpUtil = bgpUtil;
 290         this.metricProvider = metricProvider;
 291         hostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 292         portStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 293         bgpKaTime =
 294                 Integer.parseInt(getProperty(BGP_KA_TIMER_PROPERTY,
 295                         Integer.toString(DEF_BGP_KA_TIME)));
 296         bgpHoldTime =
 297                 Integer.parseInt(getProperty(BGP_HOLD_TIMER_PROPERTY,
 298                         Integer.toString(DEF_BGP_HOLD_TIME)));
 299         bgpGrRestartTime =
 300                 Integer.parseInt(getProperty(BGP_GR_RESTART_TIMER_PROPERTY,
 301                         Integer.toString(DEF_BGP_GR_TIME)));
 302         LOG.info("ConfigServer at {}:{}", hostStartup, portStartup);
 303         VtyshCli.setHostAddr(hostStartup);
 304         ClearBgpCli.setHostAddr(hostStartup);
 305         bgpUpdatesHistory = new TransactionHistory(HISTORY_LIMIT, HISTORY_THRESHOLD);
 306         bgpRouter = BgpRouter.newInstance(this::getConfig, this::isBGPEntityOwner, bgpUpdatesHistory);
 307         delayEorSeconds = Integer.parseInt(getProperty(BGP_EOR_DELAY, DEF_BGP_EOR_DELAY));
 308
 309         entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 310
 311         candidateRegistration = registerEntityCandidate(entityOwnershipService);
 312         entityListenerRegistration = registerEntityListener(entityOwnershipService);
 313
 314         LOG.info("BGP Configuration manager initialized");
 315         initer.countDown();
 316
 317         GlobalEventExecutor.INSTANCE.execute(() -> {
 318             ServiceTracker<IBgpManager, ?> tracker = null;
 319             try {
 320                 tracker = new ServiceTracker<>(bundleContext, IBgpManager.class, null);
 321                 tracker.open();
 322                 bgpManager = (IBgpManager) tracker.waitForService(TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES));
 323                 Preconditions.checkState(bgpManager != null, "IBgpManager service not found");
 324             } catch (InterruptedException e) {
 325                 throw new IllegalStateException("Error retrieving IBgpManager service", e);
 326             } finally {
 327                 if (tracker != null) {
 328                     tracker.close();
 329                 }
 330             }
 331
 332             String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
 333             if (InetAddresses.isInetAddress(getBgpSdncMipIp())) {
 334                 InetSocketAddress bgpThriftServerSocketAddr = new InetSocketAddress(getBgpSdncMipIp(),
 335                         Integer.parseInt(updatePort));
 336                 bgpThriftService = new BgpThriftService(bgpThriftServerSocketAddr, bgpManager, this);
 337                 if (isBGPEntityOwner()) {
 338                     //I am EoS owner of BGP, opening bgp thrift UPDATE-SERVER port.
 339                     LOG.info("BGP Configuration manager initialized: UPDATE-SERVER started");
 340                     bgpThriftService.start();
 341                 }
 342                 LOG.info("UPDATE server started :ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 343             } else {
 344                 LOG.error("Failed to init UPDATE server invalid ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 345             }
 346         });
 347         registerCallbacks();
 348         LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 349     }
 350
 351     public String getBgpSdncMipIp() {
 352         return getProperty(BGP_SDNC_MIP, DEF_BGP_SDNC_MIP);
 353     }
 354
 355     public long getStaleCleanupTime() {
 356         return staleCleanupTime;
 357     }
 358
 359     public void setStaleCleanupTime(long staleCleanupTime) {
 360         this.staleCleanupTime = staleCleanupTime;
 361     }
 362
 363     public long getCfgReplayEndTime() {
 364         return cfgReplayEndTime;
 365     }
 366
 367     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 368         this.cfgReplayEndTime = cfgReplayEndTime;
 369     }
 370
 371     public TransactionHistory getBgpUpdatesHistory() {
 372         return bgpUpdatesHistory;
 373     }
 374
 375     public long getCfgReplayStartTime() {
 376         return cfgReplayStartTime;
 377     }
 378
 379     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 380         this.cfgReplayStartTime = cfgReplayStartTime;
 381     }
 382
 383     public long getStaleEndTime() {
 384         return staleEndTime;
 385     }
 386
 387     public void setStaleEndTime(long staleEndTime) {
 388         this.staleEndTime = staleEndTime;
 389     }
 390
 391     public long getStaleStartTime() {
 392         return staleStartTime;
 393     }
 394
 395     public void setStaleStartTime(long staleStartTime) {
 396         this.staleStartTime = staleStartTime;
 397     }
 398
 399     private Object createListener(Class<?> cls) {
 400         try {
 401             Constructor<?> ctor = cls.getConstructor(BgpConfigurationManager.class);
 402             return ctor.newInstance(this);
 403         } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException
 404                 e) {
 405             LOG.error("Failed to create listener object", e);
 406             return null;
 407         }
 408     }
 409
 410     private void registerCallbacks() {
 411         for (Class<?> reactor : REACTORS) {
 412             Object obj = createListener(reactor);
 413             if (obj != null) {
 414                 AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 415                 dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 416                 listeners.add(dcl);
 417             }
 418         }
 419     }
 420
 421     @SuppressWarnings("checkstyle:IllegalCatch")
 422     @PreDestroy
 423     public void close() {
 424         executor.shutdown();
 425
 426         if (bgpThriftService != null) {
 427             bgpThriftService.stop();
 428             bgpThriftService = null;
 429         }
 430
 431         if (isBgpConnected()) {
 432             //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 433             bgpRouter.disconnect();
 434         }
 435
 436         if (candidateRegistration != null) {
 437             candidateRegistration.close();
 438         }
 439
 440         entityListenerRegistration.close();
 441
 442         listeners.forEach(l -> {
 443             try {
 444                 l.close();
 445             } catch (Exception e) {
 446                 LOG.warn("Error closing {}", l ,e);
 447             }
 448         });
 449
 450         LOG.info("{} close", getClass().getSimpleName());
 451     }
 452
 453     private String getProperty(String var, String def) {
 454         String property = bundleContext.getProperty(var);
 455         return property == null ? def : property;
 456     }
 457
 458     private EntityOwnershipCandidateRegistration registerEntityCandidate(
 459             final EntityOwnershipService entityOwnershipService) {
 460         try {
 461             return entityOwnershipService.registerCandidate(
 462                     new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME));
 463         } catch (CandidateAlreadyRegisteredException e) {
 464             LOG.error("failed to register bgp entity", e);
 465             return null;
 466         }
 467     }
 468
 469     private EntityOwnershipListenerRegistration registerEntityListener(
 470             final EntityOwnershipService entityOwnershipService) {
 471         return entityOwnershipService.registerListener(BGP_ENTITY_TYPE_FOR_OWNERSHIP, ownershipChange -> {
 472             LOG.info("entity owner change event fired: {}", ownershipChange);
 473
 474             if (ownershipChange.getState() == EntityOwnershipChangeState.LOCAL_OWNERSHIP_GRANTED) {
 475                 LOG.info("This PL is the Owner");
 476                 if (bgpThriftService != null) {
 477                     //opening UPDATE-SERVER port.
 478                     bgpThriftService.start();
 479                 } else {
 480                     LOG.error("I am the owner of BGP entity, but bgpThriftService is not initialized yet");
 481                 }
 482                 bgpRestarted();
 483             } else {
 484                 LOG.info("Not owner: hasOwner: {}, isOwner: {}", ownershipChange.getState().hasOwner(),
 485                         ownershipChange.getState().isOwner());
 486                 if (bgpThriftService != null && bgpThriftService.isBgpThriftServiceStarted()) {
 487                     //close the bgp Thrift Update-SERVER port opened on non-Entity Owner
 488                     bgpThriftService.stop();
 489                 }
 490                 if (isBgpConnected()) {
 491                     //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 492                     bgpRouter.disconnect();
 493                 }
 494             }
 495         });
 496     }
 497
 498     public boolean isBGPEntityOwner() {
 499         if (entityOwnershipUtils == null) {
 500             LOG.error("entityOwnershipUtils is NULL when listener callbacks fired");
 501             return false;
 502         }
 503         return entityOwnershipUtils.isEntityOwner(new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME), 0, 1);
 504     }
 505
 506     public Bgp get() {
 507         config = getConfig();
 508         return config;
 509     }
 510
 511     public class ConfigServerReactor
 512             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 513             implements ClusteredDataTreeChangeListener<ConfigServer> {
 514         private static final String YANG_OBJ = "config-server ";
 515
 516         public ConfigServerReactor() {
 517             super(ConfigServer.class, ConfigServerReactor.class);
 518         }
 519
 520         @Override
 521         protected void add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 522             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 523             if (!isBGPEntityOwner()) {
 524                 return;
 525             }
 526
 527             try {
 528                 initer.await();
 529             } catch (InterruptedException e) {
 530                 // Ignored
 531             }
 532             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 533             bgpRouter.configServerUpdated();
 534             synchronized (BgpConfigurationManager.this) {
 535                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 536                         val.getPort().intValue());
 537                 if (!res) {
 538                     LOG.error(YANG_OBJ + "Add failed; " + ADD_WARN);
 539                 }
 540             }
 541             VtyshCli.setHostAddr(val.getHost().getValue());
 542             ClearBgpCli.setHostAddr(val.getHost().getValue());
 543         }
 544
 545         @Override
 546         protected ConfigServerReactor getDataTreeChangeListener() {
 547             return ConfigServerReactor.this;
 548         }
 549
 550         @Override
 551         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 552             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 553         }
 554
 555         @Override
 556         protected void remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 557             LOG.trace("received bgp disconnect");
 558             if (!isBGPEntityOwner()) {
 559                 return;
 560             }
 561
 562             bgpRouter.configServerUpdated();
 563
 564             synchronized (BgpConfigurationManager.this) {
 565                 if (bgp_as_num != 0) {
 566                     try {
 567                         bgpRouter.stopBgp(bgp_as_num);
 568                         stopBgpCountersTask();
 569                         stopBgpAlarmsTask();
 570                     } catch (TException | BgpRouterException e) {
 571                         LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 572                     }
 573                 } else {
 574                     LOG.debug("bgp as-id is null while removing config-server");
 575                 }
 576                 bgpRouter.disconnect();
 577             }
 578         }
 579
 580         @Override
 581         protected void update(InstanceIdentifier<ConfigServer> iid,
 582                 ConfigServer oldval, ConfigServer newval) {
 583             LOG.trace("received bgp Connection update");
 584             if (!isBGPEntityOwner()) {
 585                 return;
 586             }
 587             LOG.error(YANG_OBJ + UPD_WARN);
 588         }
 589     }
 590
 591     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
 592             justification = "https://github.com/spotbugs/spotbugs/issues/811")
 593     private BgpRouter getClient(String yangObj) {
 594         if (bgpRouter == null || !bgpRouter.isBgpConnected()) {
 595             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 596             return null;
 597         }
 598         return bgpRouter;
 599     }
 600
 601     public class AsIdReactor
 602             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 603             implements ClusteredDataTreeChangeListener<AsId> {
 604
 605         private static final String YANG_OBJ = "as-id ";
 606
 607         public AsIdReactor() {
 608             super(AsId.class, AsIdReactor.class);
 609         }
 610
 611         @Override
 612         protected void add(InstanceIdentifier<AsId> iid, AsId val) {
 613             LOG.error("received bgp add asid {}", val);
 614             if (!isBGPEntityOwner()) {
 615                 return;
 616             }
 617             LOG.debug("received add router config asNum {}", val.getLocalAs());
 618             bgp_as_num = val.getLocalAs().longValue();
 619             synchronized (BgpConfigurationManager.this) {
 620                 BgpRouter br = getClient(YANG_OBJ);
 621                 if (br == null) {
 622                     LOG.debug("{} Unable to process add for asNum {}; {} {}", YANG_OBJ, val.getLocalAs(),
 623                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 624                     return;
 625                 }
 626                 bgpRestarted();
 627
 628                 startBgpCountersTask();
 629                 startBgpAlarmsTask();
 630             }
 631         }
 632
 633         @Override
 634         protected AsIdReactor getDataTreeChangeListener() {
 635             return AsIdReactor.this;
 636         }
 637
 638         @Override
 639         protected InstanceIdentifier<AsId> getWildCardPath() {
 640             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 641         }
 642
 643         @Override
 644         protected void remove(InstanceIdentifier<AsId> iid, AsId val) {
 645             LOG.error("received delete router config asNum {}", val.getLocalAs());
 646             if (!isBGPEntityOwner()) {
 647                 return;
 648             }
 649             synchronized (BgpConfigurationManager.this) {
 650                 long asNum = val.getLocalAs().toJava();
 651                 BgpRouter br = getClient(YANG_OBJ);
 652                 bgp_as_num = 0;
 653                 if (br == null) {
 654                     LOG.debug("{} Unable to process remove for asNum {}; {} {}", YANG_OBJ, asNum,
 655                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 656                     return;
 657                 }
 658                 try {
 659                     br.stopBgp(asNum);
 660                 } catch (TException | BgpRouterException e) {
 661                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 662                 }
 663
 664                 stopBgpCountersTask();
 665                 stopBgpAlarmsTask();
 666
 667                 Bgp conf = getConfig();
 668                 if (conf == null) {
 669                     LOG.error("Config Null while removing the as-id");
 670                     return;
 671                 }
 672                 LOG.debug("Removing external routes from FIB");
 673                 deleteExternalFibRoutes();
 674                 List<Neighbors> nbrs = conf.getNeighborsContainer() == null ? null
 675                         : conf.getNeighborsContainer().getNeighbors();
 676                 if (nbrs != null && nbrs.size() > 0) {
 677                     LOG.error("Tring to remove the as-id when neighbor config is already present");
 678                     for (Neighbors nbr : nbrs) {
 679                         LOG.debug("Removing Neighbor {} from Data store", nbr.getAddress().getValue());
 680                         delNeighbor(nbr.getAddress().getValue());
 681                     }
 682                 }
 683             }
 684         }
 685
 686         @Override
 687         protected void update(InstanceIdentifier<AsId> iid,
 688                 AsId oldval, AsId newval) {
 689             if (!isBGPEntityOwner()) {
 690                 return;
 691             }
 692             LOG.error(YANG_OBJ + UPD_WARN);
 693         }
 694     }
 695
 696     public class GracefulRestartReactor
 697             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 698             implements ClusteredDataTreeChangeListener<GracefulRestart> {
 699
 700         private static final String YANG_OBJ = "graceful-restart ";
 701
 702         public GracefulRestartReactor() {
 703             super(GracefulRestart.class, GracefulRestartReactor.class);
 704         }
 705
 706         @Override
 707         protected void add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 708             if (!isBGPEntityOwner()) {
 709                 return;
 710             }
 711             synchronized (BgpConfigurationManager.this) {
 712                 int stalePathTime = val.getStalepathTime().intValue();
 713                 BgpRouter br = getClient(YANG_OBJ);
 714                 if (br == null) {
 715                     LOG.error("{} Unable to add stale-path time {}; {} {}", YANG_OBJ, stalePathTime,
 716                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 717                     return;
 718                 }
 719                 try {
 720                     br.addGracefulRestart(stalePathTime);
 721                 } catch (TException | BgpRouterException e) {
 722                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 723                 }
 724             }
 725         }
 726
 727         @Override
 728         protected GracefulRestartReactor getDataTreeChangeListener() {
 729             return GracefulRestartReactor.this;
 730         }
 731
 732         @Override
 733         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 734             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 735         }
 736
 737         @Override
 738         protected void remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 739             if (!isBGPEntityOwner()) {
 740                 return;
 741             }
 742             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 743             synchronized (BgpConfigurationManager.this) {
 744                 BgpRouter br = getClient(YANG_OBJ);
 745                 if (br == null) {
 746                     LOG.error("{} Unable to delete stale-path time; {} {}", YANG_OBJ,
 747                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 748                     return;
 749                 }
 750                 try {
 751                     br.delGracefulRestart();
 752                 } catch (TException | BgpRouterException e) {
 753                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 754                 }
 755             }
 756         }
 757
 758         @Override
 759         protected void update(InstanceIdentifier<GracefulRestart> iid,
 760                 GracefulRestart oldval, GracefulRestart newval) {
 761             if (!isBGPEntityOwner()) {
 762                 return;
 763             }
 764             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 765             synchronized (BgpConfigurationManager.this) {
 766                 int stalePathTime = newval.getStalepathTime().intValue();
 767                 BgpRouter br = getClient(YANG_OBJ);
 768                 if (br == null) {
 769                     LOG.error("{} Unable to update stale-path time to {}; {} {}", YANG_OBJ, stalePathTime,
 770                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 771                     return;
 772                 }
 773                 try {
 774                     br.addGracefulRestart(stalePathTime);
 775                 } catch (TException | BgpRouterException e) {
 776                     LOG.error("{} update received exception; {}", YANG_OBJ, ADD_WARN, e);
 777                 }
 778             }
 779         }
 780     }
 781
 782     public class LoggingReactor
 783             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 784             implements ClusteredDataTreeChangeListener<Logging> {
 785
 786         private static final String YANG_OBJ = "logging ";
 787
 788         public LoggingReactor() {
 789             super(Logging.class, LoggingReactor.class);
 790         }
 791
 792         @Override
 793         protected void add(InstanceIdentifier<Logging> iid, Logging val) {
 794             if (!isBGPEntityOwner()) {
 795                 return;
 796             }
 797             synchronized (BgpConfigurationManager.this) {
 798                 BgpRouter br = getClient(YANG_OBJ);
 799                 if (br == null) {
 800                     LOG.error("{} Unable to add logging for qbgp; {} {}", YANG_OBJ,
 801                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 802                     return;
 803                 }
 804                 try {
 805                     br.setLogging(val.getFile(), val.getLevel());
 806                 } catch (TException | BgpRouterException e) {
 807                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 808                 }
 809             }
 810         }
 811
 812         @Override
 813         protected LoggingReactor getDataTreeChangeListener() {
 814             return LoggingReactor.this;
 815         }
 816
 817         @Override
 818         protected InstanceIdentifier<Logging> getWildCardPath() {
 819             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 820         }
 821
 822         @Override
 823         protected void remove(InstanceIdentifier<Logging> iid, Logging val) {
 824             if (!isBGPEntityOwner()) {
 825                 return;
 826             }
 827             LOG.debug("received remove Logging config val {}", val.getLevel());
 828             synchronized (BgpConfigurationManager.this) {
 829                 BgpRouter br = getClient(YANG_OBJ);
 830                 if (br == null) {
 831                     LOG.error("{} Unable to remove logging for qbgp; {} {}", YANG_OBJ,
 832                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 833                     return;
 834                 }
 835                 try {
 836                     br.setLogging(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
 837                 } catch (TException | BgpRouterException e) {
 838                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 839                 }
 840             }
 841         }
 842
 843         @Override
 844         protected void update(InstanceIdentifier<Logging> iid,
 845                 Logging oldval, Logging newval) {
 846             if (!isBGPEntityOwner()) {
 847                 return;
 848             }
 849             synchronized (BgpConfigurationManager.this) {
 850                 BgpRouter br = getClient(YANG_OBJ);
 851                 if (br == null) {
 852                     LOG.error("{} Unable to update logging for qbgp; {} {}", YANG_OBJ,
 853                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 854                     return;
 855                 }
 856                 try {
 857                     br.setLogging(newval.getFile(), newval.getLevel());
 858                 } catch (TException | BgpRouterException e) {
 859                     LOG.error("{} newval received exception; {}", YANG_OBJ, ADD_WARN, e);
 860                 }
 861             }
 862         }
 863     }
 864
 865     public class NeighborsReactor
 866             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 867             implements ClusteredDataTreeChangeListener<Neighbors> {
 868
 869         private static final String YANG_OBJ = "neighbors ";
 870
 871         public NeighborsReactor() {
 872             super(Neighbors.class, NeighborsReactor.class);
 873         }
 874
 875         @Override
 876         protected void add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 877             if (!isBGPEntityOwner()) {
 878                 return;
 879             }
 880             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 881             synchronized (BgpConfigurationManager.this) {
 882                 String peerIp = val.getAddress().getValue();
 883                 long as = val.getRemoteAs().toJava();
 884                 final String md5Secret = extractMd5Secret(val);
 885                 BgpRouter br = getClient(YANG_OBJ);
 886                 if (br == null) {
 887                     LOG.debug("{} Unable to process add for peer {} as {}; {} {}", YANG_OBJ, peerIp, as,
 888                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 889                     return;
 890                 }
 891                 try {
 892                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 893                     br.addNeighbor(peerIp, as, md5Secret);
 894
 895                 } catch (TException | BgpRouterException e) {
 896                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 897                 }
 898             }
 899         }
 900
 901         @Override
 902         protected NeighborsReactor getDataTreeChangeListener() {
 903             return NeighborsReactor.this;
 904         }
 905
 906         @Override
 907         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 908             return InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class);
 909         }
 910
 911         @Override
 912         protected void remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 913             if (!isBGPEntityOwner()) {
 914                 return;
 915             }
 916             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 917             synchronized (BgpConfigurationManager.this) {
 918                 String peerIp = val.getAddress().getValue();
 919                 BgpRouter br = getClient(YANG_OBJ);
 920                 if (br == null) {
 921                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 922                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 923                     return;
 924                 }
 925                 try {
 926                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 927                     br.delNeighbor(peerIp);
 928                 } catch (TException | BgpRouterException e) {
 929                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 930                 }
 931
 932                 final BgpAlarms bgpAlarms = getBgpAlarms();
 933                 if (bgpAlarms != null) {
 934                     bgpAlarms.clearBgpNbrDownAlarm(peerIp);
 935                 }
 936
 937                 if (bgpUtil.isBfdEnabled()) {
 938                     final BgpCounters bgpCounters = getBgpCounters();
 939                     if (bgpCounters != null) {
 940                         bgpCounters.clearBfdNbrCounters(peerIp);
 941                     }
 942                 }
 943             }
 944         }
 945
 946         @Override
 947         protected void update(InstanceIdentifier<Neighbors> iid,
 948                 Neighbors oldval, Neighbors newval) {
 949             if (!isBGPEntityOwner()) {
 950                 return;
 951             }
 952             //purposefully nothing to do.
 953         }
 954     }
 955
 956     public class EbgpMultihopReactor
 957             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 958             implements ClusteredDataTreeChangeListener<EbgpMultihop> {
 959
 960         private static final String YANG_OBJ = "ebgp-multihop ";
 961
 962         public EbgpMultihopReactor() {
 963             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 964         }
 965
 966         @Override
 967         protected void add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 968             if (!isBGPEntityOwner()) {
 969                 return;
 970             }
 971             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 972             synchronized (BgpConfigurationManager.this) {
 973                 String peerIp = val.getPeerIp().getValue();
 974                 BgpRouter br = getClient(YANG_OBJ);
 975                 if (br == null) {
 976                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
 977                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 978                     return;
 979                 }
 980                 try {
 981                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 982                 } catch (TException | BgpRouterException e) {
 983                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 984                 }
 985             }
 986         }
 987
 988         @Override
 989         protected EbgpMultihopReactor getDataTreeChangeListener() {
 990             return EbgpMultihopReactor.this;
 991         }
 992
 993         @Override
 994         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
 995             return InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
 996                     .child(EbgpMultihop.class);
 997         }
 998
 999         @Override
1000         protected void remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
1001             if (!isBGPEntityOwner()) {
1002                 return;
1003             }
1004             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
1005             synchronized (BgpConfigurationManager.this) {
1006                 String peerIp = val.getPeerIp().getValue();
1007                 BgpRouter br = getClient(YANG_OBJ);
1008                 if (br == null) {
1009                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1010                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1011                     return;
1012                 }
1013                 try {
1014                     br.delEbgpMultihop(peerIp);
1015                 } catch (TException | BgpRouterException e) {
1016                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1017                 }
1018             }
1019         }
1020
1021         @Override
1022         protected void update(InstanceIdentifier<EbgpMultihop> iid,
1023                 EbgpMultihop oldval, EbgpMultihop newval) {
1024             if (!isBGPEntityOwner()) {
1025                 return;
1026             }
1027             LOG.error(YANG_OBJ + UPD_WARN);
1028         }
1029     }
1030
1031     public class UpdateSourceReactor
1032             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
1033             implements ClusteredDataTreeChangeListener<UpdateSource> {
1034
1035         private static final String YANG_OBJ = "update-source ";
1036
1037         public UpdateSourceReactor() {
1038             super(UpdateSource.class, UpdateSourceReactor.class);
1039         }
1040
1041         @Override
1042         protected void add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
1043             if (!isBGPEntityOwner()) {
1044                 return;
1045             }
1046             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
1047             synchronized (BgpConfigurationManager.this) {
1048                 String peerIp = val.getPeerIp().getValue();
1049                 BgpRouter br = getClient(YANG_OBJ);
1050                 if (br == null) {
1051                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1052                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1053                     return;
1054                 }
1055                 try {
1056                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
1057                 } catch (TException | BgpRouterException e) {
1058                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1059                 }
1060             }
1061         }
1062
1063         @Override
1064         protected UpdateSourceReactor getDataTreeChangeListener() {
1065             return UpdateSourceReactor.this;
1066         }
1067
1068         @Override
1069         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
1070             return InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
1071                     .child(UpdateSource.class);
1072         }
1073
1074         @Override
1075         protected void remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
1076             if (!isBGPEntityOwner()) {
1077                 return;
1078             }
1079             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
1080             synchronized (BgpConfigurationManager.this) {
1081                 String peerIp = val.getPeerIp().getValue();
1082                 BgpRouter br = getClient(YANG_OBJ);
1083                 if (br == null) {
1084                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1085                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1086                     return;
1087                 }
1088                 try {
1089                     br.delUpdateSource(peerIp);
1090                 } catch (TException | BgpRouterException e) {
1091                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1092                 }
1093             }
1094         }
1095
1096         @Override
1097         protected void update(InstanceIdentifier<UpdateSource> iid,
1098                 UpdateSource oldval, UpdateSource newval) {
1099             if (!isBGPEntityOwner()) {
1100                 return;
1101             }
1102             LOG.error(YANG_OBJ + UPD_WARN);
1103         }
1104     }
1105
1106     public class AddressFamiliesReactor
1107             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
1108             implements ClusteredDataTreeChangeListener<AddressFamilies> {
1109
1110         private static final String YANG_OBJ = "address-families ";
1111
1112         public AddressFamiliesReactor() {
1113             super(AddressFamilies.class, AddressFamiliesReactor.class);
1114         }
1115
1116         @Override
1117         protected void add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1118             if (!isBGPEntityOwner()) {
1119                 return;
1120             }
1121             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
1122             synchronized (BgpConfigurationManager.this) {
1123                 String peerIp = val.getPeerIp().getValue();
1124                 BgpRouter br = getClient(YANG_OBJ);
1125                 if (br == null) {
1126                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1127                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1128                     return;
1129                 }
1130                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1131                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1132                 try {
1133                     br.addAddressFamily(peerIp, afi, safi);
1134                 } catch (TException | BgpRouterException e) {
1135                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1136                 }
1137             }
1138         }
1139
1140         @Override
1141         protected AddressFamiliesReactor getDataTreeChangeListener() {
1142             return AddressFamiliesReactor.this;
1143         }
1144
1145         @Override
1146         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
1147             return InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
1148                     .child(AddressFamilies.class);
1149         }
1150
1151         @Override
1152         protected void remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1153             if (!isBGPEntityOwner()) {
1154                 return;
1155             }
1156             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
1157             synchronized (BgpConfigurationManager.this) {
1158                 String peerIp = val.getPeerIp().getValue();
1159                 BgpRouter br = getClient(YANG_OBJ);
1160                 if (br == null) {
1161                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1162                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1163                     return;
1164                 }
1165                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1166                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1167                 try {
1168                     br.delAddressFamily(peerIp, afi, safi);
1169                 } catch (TException | BgpRouterException e) {
1170                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1171                 }
1172             }
1173         }
1174
1175         @Override
1176         protected void update(InstanceIdentifier<AddressFamilies> iid,
1177                 AddressFamilies oldval, AddressFamilies newval) {
1178             if (!isBGPEntityOwner()) {
1179                 return;
1180             }
1181             LOG.error(YANG_OBJ + UPD_WARN);
1182         }
1183     }
1184
1185     public class NetworksReactor
1186             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
1187             implements ClusteredDataTreeChangeListener<Networks> {
1188
1189         private static final String YANG_OBJ = "networks ";
1190
1191         public NetworksReactor() {
1192             super(Networks.class, NetworksReactor.class);
1193         }
1194
1195         @Override
1196         public NetworksReactor getDataTreeChangeListener() {
1197             return NetworksReactor.this;
1198         }
1199
1200         @Override
1201         protected void add(InstanceIdentifier<Networks> iid, Networks val) {
1202             if (!isBGPEntityOwner()) {
1203                 return;
1204             }
1205             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1206             synchronized (BgpConfigurationManager.this) {
1207                 String rd = val.getRd();
1208                 String pfxlen = val.getPrefixLen();
1209                 String nh = val.getNexthop().getValue();
1210                 BgpRouter br = getClient(YANG_OBJ);
1211                 if (br == null) {
1212                     LOG.debug("{} Unable to process add for rd {} prefix {} nexthop {}; {} {}", YANG_OBJ, rd, pfxlen,
1213                             nh, BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1214                     return;
1215                 }
1216                 Long label = val.getLabel().toJava();
1217                 int lbl = label == null ? qbgpConstants.LBL_NO_LABEL
1218                         : label.intValue();
1219                 int l3vni = val.getL3vni() == null ? qbgpConstants.LBL_NO_LABEL
1220                         : val.getL3vni().intValue();
1221                 int l2vni = val.getL2vni() == null ? qbgpConstants.LBL_NO_LABEL
1222                         : val.getL2vni().intValue();
1223
1224                 BgpControlPlaneType protocolType = val.getBgpControlPlaneType();
1225                 int ethernetTag = val.getEthtag().intValue();
1226                 String esi = val.getEsi();
1227                 String macaddress = val.getMacaddress();
1228                 EncapType encapType = val.getEncapType();
1229                 String routerMac = val.getRoutermac();
1230
1231                 try {
1232                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni, BgpUtil.convertToThriftProtocolType(protocolType),
1233                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1234                 } catch (TException | BgpRouterException e) {
1235                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1236                 }
1237             }
1238         }
1239
1240         @Override
1241         protected InstanceIdentifier<Networks> getWildCardPath() {
1242             return InstanceIdentifier.create(Bgp.class).child(NetworksContainer.class).child(Networks.class);
1243         }
1244
1245         @Override
1246         protected void remove(InstanceIdentifier<Networks> iid, Networks val) {
1247             if (!isBGPEntityOwner()) {
1248                 return;
1249             }
1250             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1251             synchronized (BgpConfigurationManager.this) {
1252                 String rd = val.getRd();
1253                 String pfxlen = val.getPrefixLen();
1254                 BgpRouter br = getClient(YANG_OBJ);
1255                 if (br == null) {
1256                     LOG.debug("{} Unable to process remove for rd {} prefix {}; {} {}", YANG_OBJ, rd, pfxlen,
1257                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1258                     return;
1259                 }
1260                 Long label = val.getLabel().toJava();
1261                 int lbl = label == null ? 0 : label.intValue();
1262                 if (rd == null && lbl > 0) {
1263                     //LU prefix is being deleted.
1264                     rd = Integer.toString(lbl);
1265                 }
1266                 try {
1267                     br.delPrefix(rd, pfxlen);
1268                 } catch (TException | BgpRouterException e) {
1269                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1270                 }
1271             }
1272         }
1273
1274         /**get the value AFI from a prefix as "x.x.x.x/x".
1275          *
1276          * @param pfxlen the prefix to get an afi
1277          * @return the afi value as you are need
1278          */
1279         public  int  testValueAFI(String pfxlen) {
1280             int afiNew = af_afi.AFI_IP.getValue();
1281             try {
1282                 String ipOnly = pfxlen.substring(0, pfxlen.lastIndexOf("/"));
1283                 java.net.Inet6Address.getByName(ipOnly);
1284                 afiNew = af_afi.AFI_IPV6.getValue();
1285             } catch (java.net.UnknownHostException e) {
1286                 //ce n'est pas de l'ipv6
1287             }
1288             return afiNew;
1289         }
1290
1291
1292         @Override
1293         protected void update(final InstanceIdentifier<Networks> iid,
1294                 final Networks oldval, final Networks newval) {
1295             if (!isBGPEntityOwner()) {
1296                 return;
1297             }
1298             if (oldval.equals(newval)) {
1299                 //Update: OLD and New values are same, no need to trigger remove/add.
1300                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1301                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1302                 return;
1303             }
1304             LOG.debug("update networks old val RD: {}, Prefix: {}, Label: {}, NH: {} "
1305                             + "new val RD: {}, Prefix: {}, Label: {}, NH: {}",
1306                     oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop(),
1307                     newval.getRd(), newval.getPrefixLen(), newval.getLabel(), newval.getNexthop());
1308             remove(iid, oldval);
1309             add(iid, newval);
1310         }
1311     }
1312
1313     static Timer timer = new Timer();
1314
1315     public class VrfsReactor
1316             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1317             implements ClusteredDataTreeChangeListener<Vrfs> {
1318
1319         private static final String YANG_OBJ = "vrfs ";
1320
1321         public VrfsReactor() {
1322             super(Vrfs.class, VrfsReactor.class);
1323         }
1324
1325         @Override
1326         protected void add(InstanceIdentifier<Vrfs> iid, Vrfs vrfs) {
1327             if (!isBGPEntityOwner()) {
1328                 return;
1329             }
1330             LOG.debug("received add Vrfs config value {}", vrfs.getRd());
1331             synchronized (BgpConfigurationManager.this) {
1332                 String rd = vrfs.getRd();
1333                 BgpRouter br = getClient(YANG_OBJ);
1334                 if (br == null) {
1335                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1336                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1337                     return;
1338                 }
1339                 try {
1340                     List<AddressFamiliesVrf> vrfAddrFamilyList = vrfs.getAddressFamiliesVrf();
1341                     for (AddressFamiliesVrf vrfAddrFamily : vrfAddrFamilyList) {
1342                         /*add to br the new vrfs arguments*/
1343                         br.addVrf(BgpUtil.getLayerType(vrfAddrFamily), rd, vrfs.getImportRts(),
1344                                 vrfs.getExportRts(), vrfAddrFamily.getAfi().toJava(), vrfAddrFamily.getSafi().toJava());
1345                     }
1346                     /*add to br the vrfs contained in mapNewAdFamily*/
1347                     List<AddressFamiliesVrf> vrfAddrFamilyListFromMap = mapNewAdFamily.get(rd);
1348                     if (vrfAddrFamilyListFromMap == null) {
1349                         return;
1350                     }
1351
1352                     for (AddressFamiliesVrf adf : vrfAddrFamilyListFromMap) {
1353                         if (vrfAddrFamilyList.contains(adf)) {
1354                             mapNewAdFamily.remove(rd);
1355                         } else  if (adf != null) {
1356
1357                             br.addVrf(BgpUtil.getLayerType(adf), rd, vrfs.getImportRts(),
1358                                     vrfs.getExportRts(), adf.getAfi().toJava(), adf.getSafi().toJava());
1359                             // remove AddressFamiliesVrf which was already added to BGP
1360                             vrfAddrFamilyListFromMap.remove(adf);
1361                             if (vrfAddrFamilyListFromMap.isEmpty()) {
1362                                 // remove Vrf entry from temp mapNewAdFamily if all its AddressFamiliesVrf was
1363                                 // added to BGP
1364                                 mapNewAdFamily.remove(rd);
1365                             }
1366                         }
1367                     }
1368                 } catch (TException | BgpRouterException e) {
1369                     LOG.error("{} get {}, Add received exception", YANG_OBJ, ADD_WARN, e);
1370                 }
1371             }
1372         }
1373
1374         @Override
1375         protected VrfsReactor getDataTreeChangeListener() {
1376             return VrfsReactor.this;
1377         }
1378
1379         @Override
1380         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1381             return InstanceIdentifier.create(Bgp.class).child(VrfsContainer.class).child(Vrfs.class);
1382         }
1383
1384         @Override
1385         protected void remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1386             if (!isBGPEntityOwner()) {
1387                 return;
1388             }
1389             LOG.debug("received remove Vrfs config val {}", val.getRd());
1390             synchronized (BgpConfigurationManager.this) {
1391                 String rd = val.getRd();
1392                 BgpRouter br = getClient(YANG_OBJ);
1393                 if (br == null) {
1394                     LOG.debug("{} Unable to process remove for rd {}; {} {}", YANG_OBJ, rd,
1395                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1396                     return;
1397                 }
1398                 try {
1399                     List<AddressFamiliesVrf> adf = mapNewAdFamily.get(rd);
1400                     adf = adf != null ? adf : new ArrayList<>();
1401                     for (AddressFamiliesVrf s : val.getAddressFamiliesVrf()) {
1402                         br.delVrf(rd, s.getAfi().toJava(), s.getSafi().toJava());
1403                         adf.remove(s);// remove in the map the vrf in waiting for advertise quagga
1404                     }
1405                     if (adf.isEmpty()) {
1406                         mapNewAdFamily.remove(rd);
1407                     }
1408                 } catch (TException | BgpRouterException e) {
1409                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1410                 }
1411             }
1412         }
1413
1414         @Override
1415         protected void update(InstanceIdentifier<Vrfs> iid,
1416                 Vrfs oldval, Vrfs newval) {
1417             if (oldval != null && newval != null) {
1418                 LOG.debug("received update Vrfs config val {}, VRFS: Update getting triggered for VRFS rd {}",
1419                         newval.getRd(), oldval.getRd());
1420             } else {
1421                 LOG.debug("received update Vrfs config val {}, from old vrf {}",
1422                         newval, oldval);
1423             }
1424             if (!isBGPEntityOwner()) {
1425                 return;
1426             }
1427
1428             List<AddressFamiliesVrf> adFamilyVrfToDel = new ArrayList<>();
1429             List<AddressFamiliesVrf> adFamilyVrfToAdd = new ArrayList<>();
1430             List<AddressFamiliesVrf> oldlistAdFamilies = new ArrayList<>();
1431             List<AddressFamiliesVrf> newlistAdFamilies = new ArrayList<>();
1432             if (oldval != null) {
1433                 oldlistAdFamilies = oldval.getAddressFamiliesVrf() == null
1434                         ? new ArrayList<>() : oldval.getAddressFamiliesVrf();
1435             }
1436             if (newval != null) {
1437                 newlistAdFamilies = newval.getAddressFamiliesVrf() == null
1438                         ? new ArrayList<>() : newval.getAddressFamiliesVrf();
1439             }
1440             /*find old AddressFamily to remove from new configuration*/
1441             for (AddressFamiliesVrf adVrf : oldlistAdFamilies) {
1442                 if (!newlistAdFamilies.contains(adVrf)) {
1443                     adFamilyVrfToDel.add(adVrf);
1444                 }
1445             }
1446             /*find new AddressFamily to add to unexisting configuration*/
1447             for (AddressFamiliesVrf adVrf : newlistAdFamilies) {
1448                 if (!oldlistAdFamilies.contains(adVrf)) {
1449                     adFamilyVrfToAdd.add(adVrf);
1450                 }
1451             }
1452             String rd = newval != null ? newval.getRd() : null;
1453             if (rd != null) {
1454                 BgpRouter br = getClient(YANG_OBJ);
1455                 if (br == null) {
1456                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1457                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1458                     return;
1459                 }
1460
1461                 for (AddressFamiliesVrf adfvrf : adFamilyVrfToAdd) {
1462                     try {
1463                         LOG.debug("call addVRf rd {} afi {} safi {}", rd, adfvrf.getAfi(), adfvrf.getSafi());
1464                         br.addVrf(BgpUtil.getLayerType(adfvrf), rd, newval.getImportRts(),
1465                                 newval.getExportRts(),adfvrf.getAfi().toJava(), adfvrf.getSafi().toJava());
1466                     } catch (TException | BgpRouterException e) {
1467                         LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1468                     }
1469                 }
1470
1471                 for (AddressFamiliesVrf adfToDel : adFamilyVrfToDel) {
1472                     try {
1473                         LOG.debug("call delVRf rd {} afi {} safi {}", rd, adfToDel.getAfi(), adfToDel.getSafi());
1474                         br.delVrf(rd, adfToDel.getAfi().toJava(), adfToDel.getSafi().toJava());
1475                     } catch (TException | BgpRouterException e) {
1476                         LOG.error("{} delVrf received exception; {}", YANG_OBJ, ADD_WARN, e);
1477                     }
1478                 }
1479             }
1480         }
1481     }
1482
1483     public class BgpReactor
1484             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1485             implements ClusteredDataTreeChangeListener<Bgp> {
1486
1487         private static final String YANG_OBJ = "Bgp ";
1488
1489         public BgpReactor() {
1490             super(Bgp.class, BgpReactor.class);
1491         }
1492
1493
1494         @Override
1495         protected void add(InstanceIdentifier<Bgp> iid, Bgp val) {
1496             LOG.debug("received add Bgp config");
1497
1498             try {
1499                 initer.await();
1500             } catch (InterruptedException e) {
1501                 // Ignored
1502             }
1503             synchronized (BgpConfigurationManager.this) {
1504                 config = val;
1505                 if (!isBGPEntityOwner()) {
1506                     return;
1507                 }
1508             }
1509         }
1510
1511         @Override
1512         protected BgpReactor getDataTreeChangeListener() {
1513             return BgpReactor.this;
1514         }
1515
1516         @Override
1517         protected InstanceIdentifier<Bgp> getWildCardPath() {
1518             return InstanceIdentifier.create(Bgp.class);
1519         }
1520
1521         @Override
1522         protected void remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1523             if (!isBGPEntityOwner()) {
1524                 return;
1525             }
1526             LOG.debug("received remove Bgp config");
1527
1528             config = null;
1529         }
1530
1531         @Override
1532         protected void update(InstanceIdentifier<Bgp> iid,
1533                 Bgp oldval, Bgp newval) {
1534             if (!isBGPEntityOwner()) {
1535                 return;
1536             }
1537
1538             config = newval;
1539         }
1540     }
1541
1542     @SuppressWarnings("deprecation")
1543     public class MultipathReactor
1544             extends AsyncDataTreeChangeListenerBase<Multipath, MultipathReactor>
1545             implements ClusteredDataTreeChangeListener<Multipath> {
1546
1547         private static final String YANG_OBJ = "multipath ";
1548
1549         public MultipathReactor() {
1550             super(Multipath.class, MultipathReactor.class);
1551         }
1552
1553
1554         @Override
1555         protected MultipathReactor getDataTreeChangeListener() {
1556             return MultipathReactor.this;
1557         }
1558
1559         @Override
1560         protected InstanceIdentifier<Multipath> getWildCardPath() {
1561             return InstanceIdentifier.create(Bgp.class).child(MultipathContainer.class).child(Multipath.class);
1562         }
1563
1564         @Override
1565         protected void remove(InstanceIdentifier<Multipath> iid, Multipath val) {
1566             executor.execute(new MultipathStatusChange(val));
1567         }
1568
1569         @Override
1570         protected void update(InstanceIdentifier<Multipath> iid, Multipath oldval, Multipath newval) {
1571             executor.execute(new MultipathStatusChange(newval));
1572         }
1573
1574         @Override
1575         protected void add(InstanceIdentifier<Multipath> key, Multipath dataObjectModification) {
1576             executor.execute(new MultipathStatusChange(dataObjectModification));
1577         }
1578
1579         class MultipathStatusChange implements Runnable {
1580
1581             Multipath multipath;
1582
1583             MultipathStatusChange(Multipath multipath) {
1584                 this.multipath = multipath;
1585             }
1586
1587             @Override
1588             public void run() {
1589                 if (isBGPEntityOwner()) {
1590                     synchronized (BgpConfigurationManager.this) {
1591
1592                         BgpRouter br = getClient(YANG_OBJ);
1593
1594                         if (br != null) {
1595                             af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1596                             af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1597
1598                             try {
1599                                 if (multipath.isMultipathEnabled()) {
1600                                     br.enableMultipath(afi, safi);
1601                                 } else {
1602                                     br.disableMultipath(afi, safi);
1603                                 }
1604                             } catch (TException | BgpRouterException e) {
1605                                 LOG.error("{} received exception", YANG_OBJ, e);
1606                             }
1607                         }
1608                     }
1609                 }
1610             }
1611
1612         }
1613
1614         @Override
1615         public void close() {
1616             super.close();
1617         }
1618     }
1619
1620     @SuppressWarnings("deprecation")
1621     public class VrfMaxpathReactor
1622             extends AsyncDataTreeChangeListenerBase<VrfMaxpath, VrfMaxpathReactor>
1623             implements ClusteredDataTreeChangeListener<VrfMaxpath> {
1624
1625         private static final String YANG_OBJ = "vrfMaxpath ";
1626
1627         public VrfMaxpathReactor() {
1628             super(VrfMaxpath.class, VrfMaxpathReactor.class);
1629         }
1630
1631
1632         @Override
1633         protected VrfMaxpathReactor getDataTreeChangeListener() {
1634             return VrfMaxpathReactor.this;
1635         }
1636
1637         @Override
1638         protected InstanceIdentifier<VrfMaxpath> getWildCardPath() {
1639             return InstanceIdentifier.create(Bgp.class).child(VrfMaxpathContainer.class).child(VrfMaxpath.class);
1640         }
1641
1642         class VrfMaxPathConfigurator implements Runnable {
1643
1644             VrfMaxpath vrfMaxpathVal;
1645
1646             VrfMaxPathConfigurator(VrfMaxpath vrfMaxPathVal) {
1647                 this.vrfMaxpathVal = vrfMaxPathVal;
1648             }
1649
1650             @Override
1651             public void run() {
1652                 if (isBGPEntityOwner()) {
1653                     synchronized (BgpConfigurationManager.this) {
1654                         BgpRouter br = getClient(YANG_OBJ);
1655                         if (br != null) {
1656                             try {
1657                                 br.multipaths(vrfMaxpathVal.getRd(), vrfMaxpathVal.getMaxpaths().toJava());
1658                                 LOG.debug("Maxpath for vrf {} is {}", vrfMaxpathVal.getRd(),
1659                                         vrfMaxpathVal.getMaxpaths());
1660                             } catch (TException | BgpRouterException e) {
1661                                 LOG.error("{} received exception", YANG_OBJ, e);
1662                             }
1663                         }
1664                     }
1665                 }
1666             }
1667         }
1668
1669         @Override
1670         protected void remove(InstanceIdentifier<VrfMaxpath> iid, VrfMaxpath vrfMaxPathVal) {
1671             if (isBGPEntityOwner()) {
1672                 synchronized (BgpConfigurationManager.this) {
1673                     BgpRouter br = getClient(YANG_OBJ);
1674                     if (br != null) {
1675                         try {
1676                             br.multipaths(vrfMaxPathVal.getRd(), BgpConstants.BGP_DEFAULT_MULTIPATH);
1677                             LOG.debug("Del Maxpath for vrf: {} ", vrfMaxPathVal.getRd());
1678                         } catch (TException | BgpRouterException e) {
1679                             LOG.error(YANG_OBJ + " del received exception:", e);
1680                         }
1681                     }
1682                 }
1683             }
1684         }
1685
1686         @Override
1687         protected void update(InstanceIdentifier<VrfMaxpath> iid,
1688                               VrfMaxpath oldval, VrfMaxpath newval) {
1689             if (!Objects.equals(oldval.getMaxpaths(), newval.getMaxpaths())) {
1690                 executor.execute(new VrfMaxPathConfigurator(newval));
1691             }
1692         }
1693
1694         @Override
1695         protected void add(InstanceIdentifier<VrfMaxpath> instanceIdentifier, VrfMaxpath vrfMaxpathVal) {
1696             executor.execute(new VrfMaxPathConfigurator(vrfMaxpathVal));
1697         }
1698
1699         @Override
1700         public void close() {
1701             super.close();
1702         }
1703     }
1704
1705     public class BfdConfigReactor
1706             extends AsyncDataTreeChangeListenerBase<BfdConfig, BfdConfigReactor>
1707             implements ClusteredDataTreeChangeListener<BfdConfig> {
1708
1709         private static final String YANG_OBJ = "BfdConfig ";
1710
1711         public BfdConfigReactor() {
1712             super(BfdConfig.class, BfdConfigReactor.class);
1713         }
1714
1715         @Override
1716         protected void add(InstanceIdentifier<BfdConfig> iid, BfdConfig val) {
1717             if (!isBGPEntityOwner()) {
1718                 return;
1719             }
1720             BgpRouter br = getClient(YANG_OBJ);
1721             LOG.debug("received bfd config: bfd enabled {} min-rx {} min-tx {} detect-mul {} mhop {}",
1722                     val.isBfdEnabled(), val.getMinRx(), val.getMinTx(),
1723                     val.getDetectMult(), val.isMultihop());
1724             if (br == null) {
1725                 LOG.debug(YANG_OBJ + "{} Unable to process add  {}",
1726                         BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1727                 return;
1728             }
1729             if (val.isBfdEnabled() == false) {
1730                 LOG.debug("BFD not enabled. Ignoring the config add");
1731                 return;
1732             }
1733             int minRx = val.getMinRx().intValue();
1734             int minTx = val.getMinTx().intValue();
1735             int detectMult = val.getDetectMult().intValue();
1736             boolean multiHop = val.isMultihop();
1737             try {
1738                 br.addBfd(detectMult, minRx, minTx,multiHop);
1739             } catch (TException | BgpRouterException e) {
1740                 LOG.error("{} get {}, Add received exception;", YANG_OBJ, ADD_WARN, e);
1741             }
1742         }
1743
1744         @Override
1745         protected BfdConfigReactor getDataTreeChangeListener() {
1746             return BfdConfigReactor.this;
1747         }
1748
1749         @Override
1750         protected InstanceIdentifier<BfdConfig> getWildCardPath() {
1751             return InstanceIdentifier.create(BfdConfig.class);
1752         }
1753
1754         @Override
1755         protected void remove(InstanceIdentifier<BfdConfig> iid, BfdConfig val) {
1756             if (!isBGPEntityOwner()) {
1757                 return;
1758             }
1759             LOG.debug("received bfd config removal");
1760             BgpRouter br = getClient(YANG_OBJ);
1761             if (br == null) {
1762                 LOG.debug("{} Unable to process del {}  {}", YANG_OBJ,
1763                         BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1764                 return;
1765             }
1766             try {
1767                 br.delBfd();
1768             } catch (TException | BgpRouterException e) {
1769                 LOG.error("{} get {}, Del received exception;", YANG_OBJ, ADD_WARN, e);
1770             }
1771
1772         }
1773
1774         @Override
1775         protected void update(InstanceIdentifier<BfdConfig> iid,
1776                               BfdConfig oldval, BfdConfig newval) {
1777             LOG.debug("received bfd config: updated oldval bfd enabled {}"
1778                     + "min-rx {} min-tx {} detect-mul {} mhop {}",
1779                     oldval.isBfdEnabled(), oldval.getMinRx(), oldval.getMinTx(),
1780                     oldval.getDetectMult(), oldval.isMultihop());
1781             LOG.debug("received bfd config: updated newval bfd enabled {}"
1782                     + "min-rx {} min-tx {} detect-mul {} mhop {}",
1783                     newval.isBfdEnabled(), newval.getMinRx(), newval.getMinTx(),
1784                     newval.getDetectMult(), newval.isMultihop());
1785             if (oldval.isBfdEnabled()) {
1786                 LOG.debug("deleting bfd config on an update");
1787                 remove(iid, oldval);
1788             }
1789             LOG.debug("adding bfd config on an update");
1790             add(iid, newval);
1791         }
1792     }
1793
1794
1795     public boolean isIpAvailable(String odlip) {
1796
1797         try {
1798             if (odlip != null) {
1799                 if ("127.0.0.1".equals(odlip)) {
1800                     return true;
1801                 }
1802                 Enumeration<NetworkInterface> networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces();
1803                 while (networkInterfaceEnumeration.hasMoreElements()) {
1804                     NetworkInterface networkInterface = networkInterfaceEnumeration.nextElement();
1805                     Enumeration<InetAddress>  inetAddressEnumeration = networkInterface.getInetAddresses();
1806                     while (inetAddressEnumeration.hasMoreElements()) {
1807                         InetAddress inetAddress = inetAddressEnumeration.nextElement();
1808                         if (odlip.equals(inetAddress.getHostAddress())) {
1809                             return true;
1810                         }
1811                     }
1812                 }
1813             }
1814         } catch (SocketException e) {
1815             // Ignored?
1816         }
1817         return false;
1818     }
1819
1820     public long getStalePathtime(int defValue, AsId asId) {
1821         long spt = 0;
1822         try {
1823             spt = getConfig().getGracefulRestart().getStalepathTime().toJava();
1824         } catch (NullPointerException e) {
1825             try {
1826                 spt = asId.getStalepathTime().toJava();
1827                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1828             } catch (NullPointerException ignore) {
1829                 LOG.trace("BGP AS id is not set using graceful");
1830                 spt = defValue;
1831             }
1832         }
1833         if (spt == 0) {
1834             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1835             spt = defValue;
1836         }
1837         return spt;
1838     }
1839
1840     public static boolean isValidConfigBgpHostPort(String bgpHost, int bgpPort) {
1841         if (!bgpHost.equals(DEF_CHOST)) {
1842             return true;
1843         } else {
1844             return false;
1845         }
1846     }
1847
1848     public synchronized void bgpRestarted() {
1849         /*
1850          * If there a thread which in the process of stale cleanup, cancel it
1851          * and start a new thread (to avoid processing same again).
1852          */
1853         if (previousReplayJobInProgress()) {
1854             cancelPreviousReplayJob();
1855         }
1856         Runnable task = () -> {
1857             try {
1858                 LOG.info("running bgp replay task ");
1859                 if (get() == null) {
1860                     String host = getConfigHost();
1861                     int port = getConfigPort();
1862                     LOG.info("connecting  to bgp host {} ", host);
1863                     bgpRouter.connect(host, port);
1864                     LOG.info("no config to push in bgp replay task ");
1865                     return;
1866                 }
1867                 setStaleStartTime(System.currentTimeMillis());
1868                 LOG.info("started creating stale fibDSWriter  map ");
1869                 createStaleFibMap();
1870                 setStaleEndTime(System.currentTimeMillis());
1871                 LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
1872                 LOG.info("started bgp config replay ");
1873                 setCfgReplayStartTime(System.currentTimeMillis());
1874                 boolean replaySucceded = replay();
1875                 setCfgReplayEndTime(System.currentTimeMillis());
1876                 LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1877                 if (replaySucceded) {
1878                     long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1879                     setStaleCleanupTime(routeSyncTime);
1880                     LOG.error("starting the stale cleanup timer: {} seconds", routeSyncTime);
1881                     routeCleanupFuture = executor.schedule(new RouteCleanup(), routeSyncTime, TimeUnit.SECONDS);
1882                 } else {
1883                     LOG.error("skipping stale cleanup, may be due to exception while replay");
1884                     staledFibEntriesMap.clear();
1885                 }
1886             } catch (InterruptedException | TimeoutException | ExecutionException eCancel) {
1887                 LOG.error("Stale Cleanup Task Cancelled", eCancel);
1888             }
1889         };
1890         lastReplayJobFt = executor.submit(task);
1891     }
1892
1893     private boolean previousReplayJobInProgress() {
1894         return ((lastReplayJobFt != null && !lastReplayJobFt.isDone())
1895                 || (routeCleanupFuture != null && !routeCleanupFuture.isDone()));
1896     }
1897
1898     private void cancelPreviousReplayJob() {
1899         try {
1900             LOG.error("cancelling already running bgp replay task");
1901             if (lastReplayJobFt != null) {
1902                 lastReplayJobFt.cancel(true);
1903                 lastReplayJobFt = null;
1904                 staledFibEntriesMap.clear();
1905             }
1906             if (routeCleanupFuture != null) {
1907                 routeCleanupFuture.cancel(true);
1908                 routeCleanupFuture = null;
1909                 staledFibEntriesMap.clear();
1910             }
1911             Thread.sleep(2000);
1912         } catch (InterruptedException e) {
1913             LOG.error("Failed to cancel previous replay job ", e);
1914         }
1915     }
1916
1917     private void doRouteSync() {
1918         for (af_afi afi : af_afi.values()) {
1919             try {
1920                 bgpRouter.initRibSync(bgpSyncHandle);
1921             } catch (BgpRouterException e) {
1922                 LOG.error("Route sync aborted, exception when initializing", e);
1923                 return;
1924             }
1925             LOG.error("Starting BGP route sync for afi {}", afi.getValue());
1926             while (bgpSyncHandle.getState() != BgpSyncHandle.DONE) {
1927                 Routes routes = null;
1928                 int noUpdates = 0;
1929                 try {
1930                     routes = bgpRouter.doRibSync(bgpSyncHandle, afi);
1931                 } catch (TException | BgpRouterException e) {
1932                     LOG.error("Route sync aborted, exception when syncing", e);
1933                     return;
1934                 }
1935                 Iterator<Update> updates = routes.getUpdatesIterator();
1936                 while (updates.hasNext()) {
1937                     noUpdates++;
1938                     Update update = updates.next();
1939                     String rd = update.getRd();
1940                     String nexthop = update.getNexthop();
1941
1942                     // TODO: decide correct label here
1943                     int label = update.getL3label();
1944                     int l2label = update.getL2label();
1945
1946                     String prefix = update.getPrefix();
1947                     int plen = update.getPrefixlen();
1948
1949
1950                     // TODO: protocol type will not be available in "update"
1951                     // use "rd" to query vrf table and obtain the protocol_type.
1952                     // Currently using PROTOCOL_EVPN as default.
1953                     onUpdatePushRoute(
1954                            protocol_type.PROTOCOL_L3VPN,
1955                            rd,
1956                            prefix,
1957                            plen,
1958                            nexthop,
1959                            update.getMacaddress(),
1960                            Uint32.valueOf(label),
1961                            Uint32.valueOf(l2label),
1962                            update.getRoutermac(),
1963                            afi);
1964                 }
1965                 LOG.error("No of updates for afi {} is {}", afi.getValue(), noUpdates);
1966             }
1967         }
1968         try {
1969             LOG.error("Ending BGP route-sync");
1970             bgpRouter.endRibSync(bgpSyncHandle);
1971         } catch (BgpRouterException e) {
1972             LOG.error("Route sync aborted, exception when ending", e);
1973         }
1974     }
1975
1976     public void addTepToElanDS(String rd, String tepIp, String mac, Uint32 l2vni) {
1977         boolean needUpdate = addToRt2TepMap(rd, tepIp, mac, l2vni);
1978         if (needUpdate) {
1979             LOG.info("Adding tepIp {} with RD {} to ELan DS", tepIp, rd);
1980             bgpUtil.addTepToElanInstance(rd, tepIp);
1981         } else {
1982             LOG.debug("Skipping the Elan update for RT2 from tep {} rd {}", tepIp, rd);
1983         }
1984     }
1985
1986     public void deleteTepfromElanDS(String rd, String tepIp, String mac) {
1987         boolean needUpdate = deleteFromRt2TepMap(rd, tepIp, mac);
1988         if (needUpdate) {
1989             LOG.info("Deleting tepIp {} with RD {} to ELan DS", tepIp, rd);
1990             bgpUtil.deleteTepFromElanInstance(rd, tepIp);
1991         } else {
1992             LOG.debug("Skipping the Elan update for RT2 withdraw from tep {} rd {}", tepIp, rd);
1993         }
1994     }
1995
1996     /* onUpdatePushRoute
1997      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1998      *  - Entry compare shall include NextHop, Label.
1999      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
2000      *  - If entry not found, add to FIB Config DS.
2001      *  - If entry found, but either Label/NextHop doesn't match.
2002      *      - Update FIB Config DS with modified values.
2003      *      - delete from Stale Map.
2004      */
2005
2006     public void onUpdatePushRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
2007                                   String macaddress, Uint32 label, Uint32 l2label, String routermac, af_afi afi) {
2008         PrefixUpdateEvent prefixUpdateEvent = new PrefixUpdateEvent(protocolType,rd,prefix,plen,nextHop,
2009                 macaddress,label,l2label,routermac,afi);
2010         bgpUpdatesHistory.addToHistory(TransactionType.ADD, prefixUpdateEvent);
2011         boolean addroute = false;
2012         boolean macupdate = false;
2013         Uint32 l3vni = Uint32.ZERO;
2014         VrfEntry.EncapType encapType = VrfEntry.EncapType.Mplsgre;
2015         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
2016             encapType = VrfEntry.EncapType.Vxlan;
2017             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
2018             if (vpnInstanceOpDataEntry != null) {
2019                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
2020                     LOG.info("Got RT2 route for RD {} l3label {} l2label {} from tep {} with mac {} remote RD {}",
2021                             vpnInstanceOpDataEntry.getVpnInstanceName(), label, l2label, nextHop, macaddress, rd);
2022                     addTepToElanDS(rd, nextHop, macaddress, l2label);
2023                     macupdate = true;
2024                 } else {
2025                     l3vni = vpnInstanceOpDataEntry.getL3vni();
2026                 }
2027             } else {
2028                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
2029                 return;
2030             }
2031         }
2032
2033         if (!staledFibEntriesMap.isEmpty()) {
2034             // restart Scenario, as MAP is not empty.
2035             Map<String, Uint32> map = staledFibEntriesMap.get(rd);
2036             if (map != null) {
2037                 String prefixNextHop = appendNextHopToPrefix(prefix + "/" + plen, nextHop);
2038                 Uint32 labelInStaleMap = map.get(prefixNextHop);
2039                 if (null == labelInStaleMap) {
2040                     // New Entry, which happened to be added during restart.
2041                     addroute = true;
2042                 } else {
2043                     map.remove(prefixNextHop);
2044                     if (isRouteModified(label, labelInStaleMap)) {
2045                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
2046                         // Existing entry, where in Label got modified during restart
2047                         addroute = true;
2048                     }
2049                 }
2050             } else {
2051                 LOG.debug("rd {} map is null while processing prefix {} ", rd, prefix);
2052                 addroute = true;
2053             }
2054         } else {
2055             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
2056             addroute = true;
2057         }
2058         if (macupdate) {
2059             LOG.info("ADD: Adding Mac Fib entry rd {} mac{} nexthop {} l2vni {}", rd, macaddress, nextHop, l2label);
2060             fibDSWriter.addMacEntryToDS(rd, macaddress, prefix, Collections.singletonList(nextHop),
2061                     encapType, l2label, routermac, RouteOrigin.BGP);
2062             LOG.info("ADD: Added Mac Fib entry rd {} prefix {} nexthop {} label {}", rd, macaddress, nextHop, l2label);
2063         } else if (addroute) {
2064             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {} afi {}",
2065                     rd, prefix, nextHop, label, afi);
2066             // TODO: modify addFibEntryToDS signature
2067             List<String> nextHopList = Collections.singletonList(nextHop);
2068             fibDSWriter.addFibEntryToDS(rd, prefix + "/" + plen, nextHopList, encapType, label, l3vni,
2069                                         routermac, RouteOrigin.BGP);
2070             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
2071             String vpnName = bgpUtil.getVpnNameFromRd(rd);
2072             if (vpnName != null) {
2073                 vpnLinkService.leakRouteIfNeeded(vpnName, prefix, nextHopList, label, RouteOrigin.BGP,
2074                                                  NwConstants.ADD_FLOW);
2075             }
2076         }
2077     }
2078
2079     public void onUpdateWithdrawRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
2080             String macaddress) {
2081         PrefixWithdrawEvent prefixWithdrawEvent = new PrefixWithdrawEvent(protocolType,rd,prefix,plen,
2082                 nextHop,macaddress);
2083         bgpUpdatesHistory.addToHistory(TransactionType.ADD, prefixWithdrawEvent);
2084         boolean macupdate = false;
2085         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
2086             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
2087             if (vpnInstanceOpDataEntry != null) {
2088                 Uint32 vni = vpnInstanceOpDataEntry.getL3vni();
2089                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
2090                     LOG.debug("Got RT2 withdraw for RD {} {} from tep {} with mac {} remote RD {}",
2091                             vpnInstanceOpDataEntry.getVpnInstanceName(), vni, nextHop, macaddress, rd);
2092                     deleteTepfromElanDS(rd, nextHop, macaddress);
2093                     LOG.debug("For rd {}. skipping fib update", rd);
2094                     macupdate = true;
2095                 }
2096             } else {
2097                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
2098                 return;
2099             }
2100         }
2101         if (macupdate) {
2102             LOG.info("Removing Mac Fib entry rd {} mac{} nexthop {} ", rd, macaddress, nextHop);
2103             fibDSWriter.removeMacEntryFromDS(rd, macaddress);
2104             LOG.info("Removed Mac Fib entry rd {} prefix {} nexthop {} ", rd, macaddress, nextHop);
2105         } else {
2106             LOG.info("REMOVE: Removing Fib entry rd {} prefix {}", rd, prefix);
2107             fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nextHop);
2108             LOG.info("REMOVE: Removed Fib entry rd {} prefix {}", rd, prefix);
2109         }
2110     }
2111
2112     //TODO: below function is for testing purpose with cli
2113     public void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
2114         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
2115         fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nexthop);
2116         String vpnName = bgpUtil.getVpnNameFromRd(rd);
2117         if (vpnName != null) {
2118             vpnLinkService.leakRouteIfNeeded(vpnName, prefix, null /*nextHopList*/, Uint32.ZERO /*INVALID_LABEL*/,
2119                                              RouteOrigin.BGP, NwConstants.DEL_FLOW);
2120         }
2121     }
2122
2123     public void peerDown(String ipAddress, long asNumber) {
2124         PeerDownEvent peerDownEvent = new PeerDownEvent(ipAddress,asNumber);
2125         bgpUpdatesHistory.addToHistory(TransactionType.ADD, peerDownEvent);
2126         List<String> tepIpList = bgpUtil.getDcgwTepConfig(ipAddress);
2127         if (tepIpList == null) {
2128             LOG.error("No Tep IP configured for DCGW {} on a peerDown", ipAddress);
2129             return;
2130         }
2131         tepIpList.forEach(tepIp -> {
2132             bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW);
2133         });
2134     }
2135
2136     public void peerUp(String ipAddress, long asNumber) {
2137         PeerUpEvent peerUpEvent = new PeerUpEvent(ipAddress,asNumber);
2138         bgpUpdatesHistory.addToHistory(TransactionType.ADD, peerUpEvent);
2139         List<String> tepIpList = bgpUtil.getDcgwTepConfig(ipAddress);
2140         if (tepIpList == null) {
2141             LOG.error("No Tep IP configured for DCGW {} on a peerUp", ipAddress);
2142             return;
2143         }
2144         tepIpList.forEach(tepIp -> {
2145             bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW);
2146         });
2147     }
2148
2149     private static boolean isRouteModified(Uint32 label, Uint32 labelInStaleMap) {
2150         return labelInStaleMap != null && !labelInStaleMap.equals(label);
2151     }
2152
2153     static class ReplayNbr {
2154         Neighbors nbr;
2155         boolean shouldRetry = false;
2156
2157         public Neighbors getNbr() {
2158             return nbr;
2159         }
2160
2161         public boolean isShouldRetry() {
2162             return shouldRetry;
2163         }
2164
2165         public void setShouldRetry(boolean retryNbr) {
2166             this.shouldRetry = retryNbr;
2167         }
2168
2169         ReplayNbr(Neighbors nbr, boolean shouldRetry) {
2170             this.nbr = nbr;
2171             this.shouldRetry = shouldRetry;
2172         }
2173     }
2174
2175     private static boolean replayNbrConfig(List<Neighbors> neighbors, BgpRouter br) {
2176         if (neighbors == null || neighbors.isEmpty()) {
2177             LOG.error("Replaying nbr configuration, received NULL list ");
2178             return true;
2179         }
2180
2181         List<ReplayNbr> replayNbrList = new ArrayList<>();
2182         for (Neighbors nbr : neighbors) {
2183             if (nbr != null) {
2184                 replayNbrList.add(new ReplayNbr(nbr, true));
2185             }
2186         }
2187         final int numberOfNbrRetries = 3;
2188         RetryOnException nbrRetry = new RetryOnException(numberOfNbrRetries);
2189         do {
2190             for (ReplayNbr replayNbr : replayNbrList) {
2191                 if (!replayNbr.isShouldRetry()) {
2192                     continue;
2193                 }
2194                 boolean replayDone = false;
2195                 LOG.debug("Replaying addNbr {}", replayNbr.getNbr().getAddress().getValue());
2196                 replayDone = false;
2197                 try {
2198                     final String md5password = extractMd5Secret(replayNbr.getNbr());
2199                     br.addNeighbor(replayNbr.getNbr().getAddress().getValue(),
2200                             replayNbr.getNbr().getRemoteAs().longValue(), md5password);
2201                     replayDone = true;
2202                 } catch (TApplicationException tae) {
2203                     LOG.debug("Replaying addNbr {}, tapplicationexception: ",
2204                             replayNbr.getNbr().getAddress().getValue(), tae);
2205                     if (tae.getType() == BgpRouterException.BGP_ERR_PEER_EXISTS) {
2206                         LOG.debug("Replaying addNbr Neighbor already present");
2207                         replayDone = true;
2208                     } else {
2209                         LOG.error("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), tae);
2210                     }
2211                 } catch (TException | BgpRouterException eNbr) {
2212                     LOG.debug("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), eNbr);
2213                 }
2214
2215                 LOG.debug("Replay addNbr {} successful", replayNbr.getNbr().getAddress().getValue());
2216
2217                 //Update Source handling
2218                 UpdateSource us = replayNbr.getNbr().getUpdateSource();
2219                 if (us != null) {
2220                     LOG.debug("Replaying updatesource {} to peer {}", us.getSourceIp().getValue(),
2221                             us.getPeerIp().getValue());
2222                     try {
2223                         br.addUpdateSource(us.getPeerIp().getValue(),
2224                                 us.getSourceIp().getValue());
2225                     } catch (TException | BgpRouterException eUs) {
2226                         LOG.debug("Replaying UpdateSource for Nbr {}, exception:",
2227                                 replayNbr.getNbr().getAddress().getValue(), eUs);
2228                     }
2229                     LOG.debug("Replay updatesource {} successful", us.getSourceIp().getValue());
2230                 }
2231                 //Ebgp Multihope
2232                 EbgpMultihop en = replayNbr.getNbr().getEbgpMultihop();
2233                 if (en != null) {
2234                     try {
2235                         br.addEbgpMultihop(en.getPeerIp().getValue(),
2236                                 en.getNhops().intValue());
2237                     } catch (TException | BgpRouterException eEbgpMhop) {
2238                         LOG.debug("Replaying EbgpMultihop for Nbr {}, exception: ",
2239                                 replayNbr.getNbr().getAddress().getValue(), eEbgpMhop);
2240                     }
2241                 }
2242
2243                 //afs
2244                 List<AddressFamilies> afs = replayNbr.getNbr().getAddressFamilies();
2245                 if (afs != null) {
2246                     for (AddressFamilies af : afs) {
2247                         af_afi afi = af_afi.findByValue(af.getAfi().intValue());
2248                         af_safi safi = af_safi.findByValue(af.getSafi().intValue());
2249                         try {
2250                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
2251                         } catch (TException | BgpRouterException eAFs) {
2252                             LOG.debug("Replaying AddressFamily for Nbr {}, exception:",
2253                                     replayNbr.getNbr().getAddress().getValue(), eAFs);
2254                         }
2255                     }
2256                 }
2257                 //replay is success --> no need to replay this nbr in next iteration.
2258                 replayNbr.setShouldRetry(replayDone ? false : true);
2259             }
2260         } while (nbrRetry.decrementAndRetry());
2261         boolean replaySuccess = true;
2262         for (ReplayNbr replayNbr : replayNbrList) {
2263             replaySuccess = replaySuccess && !replayNbr.isShouldRetry();
2264             if (replaySuccess == false) {
2265                 LOG.error("replayNbrConfig: will be cancelling stale cleanup, cfg nbr: {} Failed:",
2266                         replayNbr.getNbr().getAddress().getValue());
2267             }
2268         }
2269         return replaySuccess;
2270     }
2271
2272     public String getConfigHost() {
2273         if (config == null) {
2274             return hostStartup;
2275         }
2276         ConfigServer ts = config.getConfigServer();
2277         return ts == null ? hostStartup : ts.getHost().getValue();
2278     }
2279
2280     public int getConfigPort() {
2281         if (config == null) {
2282             return Integer.parseInt(portStartup);
2283         }
2284         ConfigServer ts = config.getConfigServer();
2285         return ts == null ? Integer.parseInt(portStartup) :
2286                 ts.getPort().intValue();
2287     }
2288
2289     public Bgp getConfig() {
2290         AtomicInteger bgpDSretryCount = new AtomicInteger(DS_RETRY_COUNT);
2291         while (0 != bgpDSretryCount.decrementAndGet()) {
2292             try {
2293                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
2294                         InstanceIdentifier.create(Bgp.class)).orNull();
2295             } catch (ReadFailedException e) {
2296                 //Config DS may not be up, so sleep for 1 second and retry
2297                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
2298                 try {
2299                     Thread.sleep(WAIT_TIME_BETWEEN_EACH_TRY_MILLIS);
2300                 } catch (InterruptedException timerEx) {
2301                     LOG.debug("WAIT_TIME_BETWEEN_EACH_TRY_MILLIS, Timer got interrupted while waiting for"
2302                             + "config DS availability", timerEx);
2303                 }
2304             }
2305         }
2306         LOG.error("failed to get bgp config");
2307         return null;
2308     }
2309
2310     @SuppressWarnings("checkstyle:IllegalCatch")
2311     public synchronized boolean replay() throws InterruptedException, TimeoutException, ExecutionException {
2312         boolean replaySucceded = true;
2313         boolean doRouteSync = false;
2314         String host = getConfigHost();
2315         int port = getConfigPort();
2316         LOG.error("connecting  to bgp host {} ", host);
2317         boolean res = bgpRouter.connect(host, port);
2318         if (!res) {
2319             LOG.error("Cannot connect to BGP config server at {} {}", host, port);
2320             return replaySucceded;
2321         }
2322         config = getConfig();
2323         if (config == null) {
2324             LOG.error("bgp config is empty nothing to push to bgp");
2325             return replaySucceded;
2326         }
2327         BgpRouter br = bgpRouter;
2328         AsId asId = config.getAsId();
2329         if (asId == null) {
2330             LOG.error("bgp as-id is null");
2331             return replaySucceded;
2332         }
2333         long asNum = asId.getLocalAs().toJava();
2334         IpAddress routerId = asId.getRouterId();
2335         String rid = routerId == null ? "" : routerId.stringValue();
2336         int stalepathTime = (int) getStalePathtime(bgpGrRestartTime, config.getAsId());
2337         boolean announceFbit = true;
2338         boolean replayDone = false;
2339         final int numberOfStartBgpRetries = 3;
2340         RetryOnException startBgpRetry = new RetryOnException(numberOfStartBgpRetries);
2341         do {
2342             try {
2343                 LOG.debug("Replaying BGPConfig ");
2344                 br.startBgp(asNum, rid, bgpKaTime, bgpHoldTime, stalepathTime, announceFbit);
2345                 LOG.debug("Replay BGPConfig successful");
2346                 replayDone = true;
2347                 break;
2348             } catch (BgpRouterException bre) {
2349                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
2350                     LOG.debug("Starting the routesync for exception", bre);
2351                     startBgpRetry.errorOccured();
2352                     if (!startBgpRetry.shouldRetry()) {
2353                         doRouteSync = true;
2354                         replayDone = true;
2355                     }
2356                 } else {
2357                     LOG.error("Replay: startBgp() received exception error {} : ",
2358                             bre.getErrorCode(), bre);
2359                     startBgpRetry.errorOccured();
2360                 }
2361             } catch (TApplicationException tae) {
2362                 if (tae.getType() == BgpRouterException.BGP_ERR_ACTIVE) {
2363                     LOG.debug("Starting the routesync for exception", tae);
2364                     startBgpRetry.errorOccured();
2365                     if (!startBgpRetry.shouldRetry()) {
2366                         doRouteSync = true;
2367                         replayDone = true;
2368                     }
2369                 } else if (tae.getType() == BgpRouterException.BGP_ERR_COMMON_FAILURE) {
2370                     LOG.debug("Starting the routesync for AS-ID started exception", tae);
2371                     startBgpRetry.errorOccured();
2372                     if (!startBgpRetry.shouldRetry()) {
2373                         doRouteSync = true;
2374                         replayDone = true;
2375                     }
2376                 } else {
2377                     LOG.error("Replay: startBgp() received exception type {}: ",
2378                             tae.getType(), tae);
2379                     startBgpRetry.errorOccured();
2380                 }
2381             } catch (Exception e) {
2382                 //not unusual. We may have restarted & BGP is already on
2383                 LOG.error("Replay:startBgp() received exception: ", e);
2384                 startBgpRetry.errorOccured();
2385             }
2386         } while (startBgpRetry.shouldRetry());
2387
2388         replaySucceded = replayDone;
2389
2390         startBgpCountersTask();
2391         startBgpAlarmsTask();
2392
2393         /*
2394          * commenting this due to a bug with QBGP. Will uncomment once QBGP fix is done.
2395          * This wont have any functional impacts
2396          */
2397         //try {
2398         //    br.delayEOR(delayEorSeconds);
2399         //} catch (TException | BgpRouterException e) {
2400         //    LOG.error("Replay: delayEOR() number of seconds to wait for EOR from ODL:", e);
2401         //}
2402
2403         BfdConfig bfdConfig = bgpUtil.getBfdConfig();
2404         if (bfdConfig != null) {
2405             if (bfdConfig.isBfdEnabled()) {
2406                 LOG.debug("Replaying bfd config min-rx {} min-tx {} detect-mul {} mhop {}",
2407                         bfdConfig.getMinRx(), bfdConfig.getMinTx(),
2408                         bfdConfig.getDetectMult(), bfdConfig.isMultihop());
2409                 try {
2410                     br.addBfd(bfdConfig.getDetectMult().intValue(), bfdConfig.getMinRx().intValue(),
2411                             bfdConfig.getMinTx().intValue(), bfdConfig.isMultihop());
2412                 } catch (TApplicationException tae) {
2413                     if (tae.getType() == BgpRouterException.BGP_ERR_PEER_EXISTS) {
2414                         LOG.debug("Replay:addBfd() received exception", tae);
2415                     } else {
2416                         LOG.error("Replay:addBfd() received exception", tae);
2417                     }
2418                 } catch (TException | BgpRouterException e) {
2419                     LOG.error("Replay:addBfd() received exception", e);
2420                 }
2421             }
2422         }
2423
2424         List<Neighbors> neighbors = config.getNeighborsContainer() == null ? null
2425                 : config.getNeighborsContainer().getNeighbors();
2426         if (neighbors != null) {
2427             LOG.error("configuring existing Neighbors present for replay total neighbors {}", neighbors.size());
2428             boolean neighborConfigReplayResult = replayNbrConfig(neighbors, br);
2429             if (neighborConfigReplayResult == false) {
2430                 replaySucceded = false;
2431             }
2432         } else {
2433             LOG.error("no Neighbors present for replay config ");
2434         }
2435
2436         Logging logging = config.getLogging();
2437         if (logging != null) {
2438             try {
2439                 br.setLogging(logging.getFile(), logging.getLevel());
2440             } catch (TException | BgpRouterException e) {
2441                 LOG.error("Replay:setLogging() received exception", e);
2442             }
2443         }
2444
2445         GracefulRestart gracefulRestart = config.getGracefulRestart();
2446         bgpGrRestartTime = ((gracefulRestart != null)
2447                 ? gracefulRestart.getStalepathTime().intValue() : bgpGrRestartTime);
2448         try {
2449             br.addGracefulRestart(bgpGrRestartTime);
2450         } catch (Exception e) {
2451             LOG.error("Replay:addGr() received exception: ", e);
2452         }
2453         List<Vrfs> vrfs = config.getVrfsContainer() == null ? null
2454                 : config.getVrfsContainer().getVrfs();
2455         if (vrfs == null) {
2456             vrfs = new ArrayList<>();
2457         }
2458         for (Vrfs vrf : vrfs) {
2459             for (AddressFamiliesVrf adf : vrf.getAddressFamiliesVrf()) {
2460                 try {
2461                     br.addVrf(BgpUtil.getLayerType(adf), vrf.getRd(), vrf.getImportRts(),
2462                             vrf.getExportRts(), adf.getAfi().toJava(), adf.getSafi().toJava());
2463                 } catch (TException | BgpRouterException e) {
2464                     LOG.error("Replay:addVrf() received exception", e);
2465                 }
2466             }
2467         }
2468
2469
2470         List<Networks> ln = config.getNetworksContainer() == null ? null
2471                 : config.getNetworksContainer().getNetworks();
2472         if (ln != null) {
2473             for (Networks net : ln) {
2474                 String rd = net.getRd();
2475                 String pfxlen = net.getPrefixLen();
2476                 String nh = net.getNexthop().getValue();
2477                 Long label = net.getLabel().toJava();
2478                 int lbl = label == null ? 0 : label.intValue();
2479                 int l3vni = net.getL3vni() == null ? 0 : net.getL3vni().intValue();
2480                 int l2vni = net.getL2vni() == null ? 0 : net.getL2vni().intValue();
2481                 if (rd == null && lbl > 0) {
2482                     //LU prefix is being deleted.
2483                     rd = Integer.toString(lbl);
2484                 }
2485
2486                 BgpControlPlaneType protocolType = net.getBgpControlPlaneType();
2487                 int ethernetTag = net.getEthtag().intValue();
2488                 String esi = net.getEsi();
2489                 String macaddress = net.getMacaddress();
2490                 EncapType encapType = net.getEncapType();
2491                 String routerMac = net.getRoutermac();
2492
2493                 try {
2494                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni,
2495                             BgpUtil.convertToThriftProtocolType(protocolType),
2496                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
2497                 } catch (TException | BgpRouterException e) {
2498                     LOG.error("Replay:addPfx() received exception", e);
2499                 }
2500             }
2501         }
2502
2503
2504         List<Multipath> multipaths = config.getMultipathContainer() == null ? null
2505                 : config.getMultipathContainer().getMultipath();
2506
2507         if (multipaths != null) {
2508             for (Multipath multipath : multipaths) {
2509                 if (multipath != null) {
2510                     af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
2511                     af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
2512
2513                     try {
2514                         if (multipath.isMultipathEnabled()) {
2515                             br.enableMultipath(afi, safi);
2516                         } else {
2517                             br.disableMultipath(afi, safi);
2518                         }
2519                     } catch (TException | BgpRouterException e) {
2520                         LOG.info("Replay:multipaths() received exception", e);
2521                     }
2522                 }
2523             }
2524         }
2525         List<VrfMaxpath> vrfMaxpaths = config.getVrfMaxpathContainer() == null ? null
2526                 : config.getVrfMaxpathContainer().getVrfMaxpath();
2527         if (vrfMaxpaths != null) {
2528             for (VrfMaxpath vrfMaxpath : vrfMaxpaths) {
2529                 try {
2530                     br.multipaths(vrfMaxpath.getRd(), vrfMaxpath.getMaxpaths().toJava());
2531                 } catch (TException | BgpRouterException e) {
2532                     LOG.info("Replay:vrfMaxPath() received exception", e);
2533                 }
2534             }
2535         }
2536
2537         //send End of Rib Marker to Qthriftd.
2538         final int numberOfEORRetries = 3;
2539         RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
2540         do {
2541             try {
2542                 br.sendEOR();
2543                 LOG.debug("Replay sendEOR() successful");
2544                 break;
2545             } catch (Exception e) {
2546                 eorRetry.errorOccured();
2547                 LOG.error("Replay:sedEOR() received exception:", e);
2548             }
2549         } while (eorRetry.shouldRetry());
2550
2551         if (doRouteSync) {
2552             LOG.debug("starting route sync for Thrift BGP_ERR_COMMON_FAILURE exception "
2553                       + "happened earlier");
2554             doRouteSync();
2555         }
2556
2557         return replaySucceded;
2558     }
2559
2560     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
2561         bgpUtil.update(iid, dto);
2562     }
2563
2564     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
2565         bgpUtil.delete(iid);
2566     }
2567
2568     public void startConfig(String bgpHost, int thriftPort) {
2569         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2570                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2571         InstanceIdentifier<ConfigServer> iid = iib.build();
2572         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
2573         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
2574                 .setPort((long) thriftPort).build();
2575         update(iid, dto);
2576     }
2577
2578     public void startBgp(long as, String routerId, int spt, boolean fbit) {
2579         IpAddress rid = routerId == null ? null : IpAddressBuilder.getDefaultInstance(routerId);
2580         Long staleTime = (long) spt;
2581         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2582                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2583         InstanceIdentifier<AsId> iid = iib.build();
2584         AsId dto = new AsIdBuilder().setLocalAs(as)
2585                 .setRouterId(rid)
2586                 .setStalepathTime(staleTime)
2587                 .setAnnounceFbit(fbit).build();
2588         update(iid, dto);
2589     }
2590
2591     public void startBfd(long detectMult, long minRx, long minTx, boolean multiHop) {
2592         InstanceIdentifier.InstanceIdentifierBuilder<BfdConfig> iib =
2593                 InstanceIdentifier.builder(BfdConfig.class);
2594         InstanceIdentifier<BfdConfig> iid = iib.build();
2595         BfdConfig dto = new BfdConfigBuilder()
2596                 .setBfdEnabled(true)
2597                 .setMultihop(multiHop)
2598                 .setMinRx(minRx)
2599                 .setMinTx(minTx)
2600                 .setDetectMult(detectMult)
2601                 .build();
2602         update(iid, dto);
2603     }
2604
2605     public void addDcgwTep(String dcgwIp, String tepIp) {
2606         InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2607                 InstanceIdentifier.builder(Bgp.class)
2608                         .child(DcgwTepList.class)
2609                         .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2610         InstanceIdentifier<DcgwTep> iid = iib.build();
2611         ArrayList<String> tepList = new ArrayList<String>();
2612         tepList.add(tepIp);
2613         DcgwTep dto = new DcgwTepBuilder().setDcGwIp(dcgwIp).setTepIps(tepList)
2614                 .build();
2615         update(iid, dto);
2616         bgpUtil.removeOrUpdateLBGroups(tepIp,NwConstants.MOD_FLOW);
2617     }
2618
2619     public void addLogging(String fileName, String logLevel) {
2620         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2621                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2622         InstanceIdentifier<Logging> iid = iib.build();
2623         Logging dto = new LoggingBuilder().setFile(fileName)
2624                 .setLevel(logLevel).build();
2625         update(iid, dto);
2626     }
2627
2628     public void addGracefulRestart(int staleTime) {
2629         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2630                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
2631         InstanceIdentifier<GracefulRestart> iid = iib.build();
2632         GracefulRestart dto = new GracefulRestartBuilder()
2633                 .setStalepathTime((long) staleTime).build();
2634         update(iid, dto);
2635     }
2636
2637     public void addNeighbor(
2638             String nbrIp, long remoteAs, @Nullable final TcpMd5SignaturePasswordType md5Secret) {
2639         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2640         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2641                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2642                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2643         InstanceIdentifier<Neighbors> iid = iib.build();
2644         TcpSecurityOption tcpSecOption = null;
2645         if (md5Secret != null) {
2646             tcpSecOption = new TcpMd5SignatureOptionBuilder().setTcpMd5SignaturePassword(md5Secret).build();
2647         } // else let tcpSecOption be null
2648         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
2649                 .setRemoteAs(remoteAs).setTcpSecurityOption(tcpSecOption).build();
2650         update(iid, dto);
2651     } // public addNeighbor(nbrIp, remoteAs, md5Secret)
2652
2653     public void addUpdateSource(String nbrIp, String srcIp) {
2654         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2655         Ipv4Address srcAddr = new Ipv4Address(srcIp);
2656         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2657                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2658                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2659                         .child(UpdateSource.class);
2660         InstanceIdentifier<UpdateSource> iid = iib.build();
2661         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
2662                 .setSourceIp(srcAddr).build();
2663         update(iid, dto);
2664     }
2665
2666     public void addEbgpMultihop(String nbrIp, int hops) {
2667         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2668         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2669                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2670                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2671                         .child(EbgpMultihop.class);
2672         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2673         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
2674                 .setNhops((long) hops).build();
2675         update(iid, dto);
2676     }
2677
2678     public void addAddressFamily(String nbrIp, int afi, int safi) {
2679         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2680         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2681                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2682                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2683                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2684         InstanceIdentifier<AddressFamilies> iid = iib.build();
2685         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
2686                 .setAfi((long) afi).setSafi((long) safi).build();
2687         update(iid, dto);
2688     }
2689
2690     public void addPrefix(String rd, String macAddress, String pfx, List<String> nhList,
2691               VrfEntry.EncapType encapType, Uint32 lbl, Uint32 l3vni, Uint32 l2vni, String gatewayMac) {
2692         for (String nh : nhList) {
2693             Ipv4Address nexthop = nh != null ? new Ipv4Address(nh) : null;
2694             Uint32 label = lbl;
2695             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
2696                     .child(NetworksContainer.class)
2697                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
2698             NetworksBuilder networksBuilder = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
2699                                                 .setLabel(label).setEthtag(BgpConstants.DEFAULT_ETH_TAG);
2700             buildVpnEncapSpecificInfo(networksBuilder, encapType, label, l3vni, l2vni, macAddress, gatewayMac);
2701             update(iid, networksBuilder.build());
2702         }
2703     }
2704
2705     private static void buildVpnEncapSpecificInfo(NetworksBuilder builder, VrfEntry.EncapType encapType, Uint32 label,
2706                                                   Uint32 l3vni, Uint32 l2vni, String macAddress, String gatewayMac) {
2707         if (encapType.equals(VrfEntry.EncapType.Mplsgre)) {
2708             builder.setLabel(label).setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLL3VPN)
2709                     .setEncapType(EncapType.GRE);
2710         } else {
2711             builder.setL3vni(l3vni).setL2vni(l2vni).setMacaddress(macAddress).setRoutermac(gatewayMac)
2712                     .setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLEVPN).setEncapType(EncapType.VXLAN);
2713         }
2714     }
2715
2716     // TODO: add LayerType as arg - supports command
2717     public void addVrf(String rd, List<String> irts, List<String> erts, AddressFamily addressFamily) {
2718         Vrfs vrf = bgpUtil.getVrfFromRd(rd);
2719         List<AddressFamiliesVrf> adfList = new ArrayList<>(1);
2720         if (vrf != null) {
2721             adfList = vrf.getAddressFamiliesVrf();
2722         }
2723         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2724         if (addressFamily.equals(AddressFamily.IPV4)) {
2725             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2726             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2727         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2728             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2729             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2730         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2731             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2732             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2733         }
2734         AddressFamiliesVrf adf = adfBuilder.build();
2735         adfList.add(adf);
2736         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib = InstanceIdentifier.builder(Bgp.class)
2737                 .child(VrfsContainer.class)
2738                 .child(Vrfs.class, new VrfsKey(rd));
2739         InstanceIdentifier<Vrfs> iid = iib.build();
2740         Vrfs dto = new VrfsBuilder().setRd(rd).setImportRts(irts)
2741             .setExportRts(erts).setAddressFamiliesVrf(adfList).build();
2742
2743         List<AddressFamiliesVrf> listAdFamilies = mapNewAdFamily.get(rd);
2744         if (listAdFamilies != null) {
2745             listAdFamilies.add(adf);
2746         } else {
2747             mapNewAdFamily.put(rd, adfList);
2748         }
2749
2750         try {
2751             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
2752         } catch (TransactionCommitFailedException e) {
2753             LOG.error("Error adding VRF to datastore", e);
2754             throw new RuntimeException(e);
2755         }
2756
2757         // enable multipath by default in all VRFs
2758         setMultipaths(rd, BgpConstants.BGP_DEFAULT_MULTIPATH);
2759     }
2760
2761     public void stopConfig() {
2762         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2763                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2764         InstanceIdentifier<ConfigServer> iid = iib.build();
2765         delete(iid);
2766     }
2767
2768     public void stopBgp() {
2769         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2770                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2771         InstanceIdentifier<AsId> iid = iib.build();
2772         delete(iid);
2773     }
2774
2775     public void stopBfd() {
2776         InstanceIdentifier.InstanceIdentifierBuilder<BfdConfig> iib =
2777                 InstanceIdentifier.builder(BfdConfig.class);
2778         InstanceIdentifier<BfdConfig> iid = iib.build();
2779         delete(iid);
2780     }
2781
2782     public void delDcgwTep(String dcgwIp, String tepIp) {
2783         if (tepIp == null) {
2784             InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2785                     InstanceIdentifier.builder(Bgp.class)
2786                             .child(DcgwTepList.class)
2787                             .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2788             InstanceIdentifier<DcgwTep> iid = iib.build();
2789             delete(iid);
2790         } else {
2791             InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2792                     InstanceIdentifier.builder(Bgp.class)
2793                             .child(DcgwTepList.class)
2794                             .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2795             InstanceIdentifier<DcgwTep> iid = iib.build();
2796             List<String> tepIpList = bgpUtil.getDcgwTepConfig(dcgwIp);
2797             if (tepIpList == null) {
2798                 LOG.error("No Tep IP configured for DCGW {} on deleting the dcgwtep", dcgwIp);
2799                 return;
2800             }
2801             List<String> newTepIpList = new ArrayList<String>();
2802             tepIpList.forEach(tep -> {
2803                 if (!tep.equals(tepIp)) {
2804                     newTepIpList.add(tep);
2805                 }
2806             });
2807             DcgwTep dto = new DcgwTepBuilder().setDcGwIp(dcgwIp).setTepIps(newTepIpList)
2808                     .build();
2809             try {
2810                 SingleTransactionDataBroker.syncWrite(dataBroker,
2811                         LogicalDatastoreType.CONFIGURATION, iid, dto);
2812             } catch (TransactionCommitFailedException e) {
2813                 LOG.error("delDcgwTep: Error deleting DCGW Tep", e);
2814                 throw new RuntimeException(e);
2815             }
2816         }
2817     }
2818
2819     public void delLogging() {
2820         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2821                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2822         InstanceIdentifier<Logging> iid = iib.build();
2823         delete(iid);
2824     }
2825
2826     public void delGracefulRestart() {
2827         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2828                 InstanceIdentifier.builder(Bgp.class)
2829                         .child(GracefulRestart.class);
2830         InstanceIdentifier<GracefulRestart> iid = iib.build();
2831         delete(iid);
2832     }
2833
2834     public void delNeighbor(String nbrIp) {
2835         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2836         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2837                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2838                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2839         InstanceIdentifier<Neighbors> iid = iib.build();
2840         delete(iid);
2841     }
2842
2843     public void delUpdateSource(String nbrIp) {
2844         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2845         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2846                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2847                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2848                         .child(UpdateSource.class);
2849         InstanceIdentifier<UpdateSource> iid = iib.build();
2850         delete(iid);
2851     }
2852
2853     public void delEbgpMultihop(String nbrIp) {
2854         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2855         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2856                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2857                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2858                         .child(EbgpMultihop.class);
2859         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2860         delete(iid);
2861     }
2862
2863     public void delAddressFamily(String nbrIp, int afi, int safi) {
2864         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2865         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2866                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2867                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2868                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2869         InstanceIdentifier<AddressFamilies> iid = iib.build();
2870         delete(iid);
2871     }
2872
2873     public void delPrefix(String rd, String pfx) {
2874         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
2875                 InstanceIdentifier.builder(Bgp.class).child(NetworksContainer.class)
2876                         .child(Networks.class, new NetworksKey(pfx, rd));
2877         InstanceIdentifier<Networks> iid = iib.build();
2878         delete(iid);
2879     }
2880
2881     public boolean delVrf(String rd, AddressFamily addressFamily) {
2882         if (addressFamily == null) {
2883             LOG.error("delVrf: vrf {}, addressFamily invalid", rd);
2884             return false;
2885         }
2886         delMultipaths(rd);
2887         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2888         if (addressFamily.equals(AddressFamily.IPV4)) {
2889             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2890             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2891         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2892             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2893             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2894         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2895             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2896             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2897         }
2898         LOG.debug("delVrf: Received Delete VRF : rd:{}, address family: {} {}", rd,
2899                 adfBuilder.getAfi(), adfBuilder.getSafi());
2900
2901         Vrfs vrfOriginal = bgpUtil.getVrfFromRd(rd);
2902         if (vrfOriginal == null) {
2903             LOG.error("delVrf: no vrf with existing rd {}. step aborted", rd);
2904             return false;
2905         }
2906
2907         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2908                 InstanceIdentifier.builder(Bgp.class).child(VrfsContainer.class)
2909                         .child(Vrfs.class, new VrfsKey(rd));
2910
2911         InstanceIdentifier<Vrfs> iid = iib.build();
2912
2913         @SuppressWarnings("static-access")
2914         InstanceIdentifier<Bgp> iid6 =  iid.builder(Bgp.class).build()
2915                 .child(MultipathContainer.class)
2916                 .child(Multipath.class, new MultipathKey(adfBuilder.getAfi(), adfBuilder.getSafi())).create(Bgp.class);
2917         InstanceIdentifierBuilder<Vrfs> iib3 =
2918                 iid6.child(VrfsContainer.class).child(Vrfs.class, new VrfsKey(rd)).builder();
2919         InstanceIdentifier<Vrfs> iidFinal = iib3.build();
2920
2921         //** update or delete the vrfs with the rest of AddressFamilies already present in the last list
2922         AddressFamiliesVrf adfToDel = adfBuilder.build();
2923         List<AddressFamiliesVrf> adfListOriginal = new ArrayList<>(vrfOriginal.nonnullAddressFamiliesVrf());
2924         List<AddressFamiliesVrf> adfListToRemoveFromOriginal = new ArrayList<>();
2925         adfListOriginal.forEach(adf -> {
2926             if (adf.equals(adfToDel)) {
2927                 adfListToRemoveFromOriginal.add(adfToDel);
2928                 return;
2929             }
2930         });
2931         for (AddressFamiliesVrf adfToRemove : adfListToRemoveFromOriginal) {
2932             adfListOriginal.remove(adfToRemove);
2933             try {
2934                 SingleTransactionDataBroker.syncWrite(dataBroker,
2935                         LogicalDatastoreType.CONFIGURATION, iid, vrfOriginal);
2936             } catch (TransactionCommitFailedException e) {
2937                 LOG.error("delVrf: Error updating VRF to datastore", e);
2938                 throw new RuntimeException(e);
2939             }
2940         }
2941         if (adfListOriginal.isEmpty()) {
2942             LOG.debug("delVrf: delete iid: {}", iidFinal);
2943             delete(iidFinal);
2944             return true;
2945         }
2946         // not all is removed
2947         return false;
2948     }
2949
2950     public void setMultipathStatus(af_afi afi, af_safi safi, boolean enable) {
2951         long lafi = afi.getValue();
2952         long lsafi = safi.getValue();
2953
2954         InstanceIdentifier.InstanceIdentifierBuilder<Multipath> iib =
2955                 InstanceIdentifier
2956                         .builder(Bgp.class).child(MultipathContainer.class)
2957                         .child(Multipath.class,
2958                                 new MultipathKey(Long.valueOf(afi.getValue()), Long.valueOf(safi.getValue())));
2959
2960         Multipath dto = new MultipathBuilder().setAfi(lafi).setSafi(lsafi).setMultipathEnabled(enable).build();
2961         update(iib.build(), dto);
2962     }
2963
2964     public void setMultipaths(String rd, int maxpath) {
2965         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
2966                 InstanceIdentifier
2967                         .builder(Bgp.class).child(VrfMaxpathContainer.class)
2968                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
2969
2970         VrfMaxpath dto = new VrfMaxpathBuilder().setRd(rd).setMaxpaths(maxpath).build();
2971         update(iib.build(), dto);
2972     }
2973
2974     public void delMultipaths(String rd) {
2975         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
2976                 InstanceIdentifier.builder(Bgp.class).child(VrfMaxpathContainer.class)
2977                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
2978         InstanceIdentifier<VrfMaxpath> iid = iib.build();
2979         delete(iid);
2980     }
2981
2982     /*
2983     * Remove Stale Marked Routes after timer expiry.
2984     */
2985     private class RouteCleanup implements Callable<Integer> {
2986
2987         @Override
2988         public Integer call() {
2989             totalCleared = 0;
2990             try {
2991                 if (staledFibEntriesMap.isEmpty()) {
2992                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
2993                 } else {
2994                     for (String rd : staledFibEntriesMap.keySet()) {
2995                         if (Thread.interrupted()) {
2996                             return 0;
2997                         }
2998                         Map<String, Uint32> map = staledFibEntriesMap.get(rd);
2999                         if (map != null) {
3000                             for (String key : map.keySet()) {
3001                                 if (Thread.interrupted()) {
3002                                     return 0;
3003                                 }
3004                                 String prefix = extractPrefix(key);
3005                                 String nextHop = extractNextHop(key);
3006                                 totalCleared++;
3007                                 LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}, nextHop:{}",
3008                                         rd, prefix, nextHop);
3009                                 fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix, nextHop);
3010                             }
3011                         }
3012                     }
3013                 }
3014             } finally {
3015                 staledFibEntriesMap.clear();
3016             }
3017             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
3018             return 0;
3019         }
3020     }
3021
3022     /*
3023      * BGP restart scenario, ODL-BGP manager was/is running.
3024      * On re-sync notification, Get a copy of FIB database.
3025      */
3026     public void createStaleFibMap() {
3027         totalStaledCount = 0;
3028         try {
3029             staledFibEntriesMap.clear();
3030             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
3031
3032             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
3033                     LogicalDatastoreType.CONFIGURATION, id);
3034             if (fibEntries.isPresent()) {
3035                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
3036                 for (VrfTables vrfTable : staleVrfTables) {
3037                     Map<String, Uint32> staleFibEntMap = new HashMap<>();
3038                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
3039                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
3040                             //Stale marking and cleanup is only meant for the routes learned through BGP.
3041                             continue;
3042                         }
3043                         if (Thread.interrupted()) {
3044                             break;
3045                         }
3046                         totalStaledCount++;
3047                         //Create MAP from staleVrfTables.
3048                         vrfEntry.getRoutePaths()
3049                                 .forEach(
3050                                     routePath -> staleFibEntMap.put(
3051                                             appendNextHopToPrefix(vrfEntry.getDestPrefix(),
3052                                                     routePath.getNexthopAddress()), routePath.getLabel()));
3053                     }
3054                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), staleFibEntMap);
3055                 }
3056             } else {
3057                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
3058             }
3059         } catch (ReadFailedException e) {
3060             LOG.error("createStaleFibMap:: error ", e);
3061         }
3062         LOG.error("created {} staled entries ", totalStaledCount);
3063     }
3064
3065     /*
3066      * BGP config remove scenario, Need to remove all the
3067      * external routes from FIB.
3068      */
3069     public void deleteExternalFibRoutes() {
3070         totalExternalRoutes = 0;
3071         totalExternalMacRoutes = 0;
3072         try {
3073             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
3074
3075             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
3076                     LogicalDatastoreType.CONFIGURATION, id);
3077             if (fibEntries.isPresent()) {
3078                 if (fibEntries.get().getVrfTables() == null) {
3079                     LOG.error("deleteExternalFibRoutes::getVrfTables is null");
3080                     return;
3081                 }
3082                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
3083                 for (VrfTables vrfTable : staleVrfTables) {
3084                     String rd = vrfTable.getRouteDistinguisher();
3085                     if (vrfTable.getVrfEntry() != null) {
3086                         for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
3087                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
3088                                 //route cleanup is only meant for the routes learned through BGP.
3089                                 continue;
3090                             }
3091                             totalExternalRoutes++;
3092                             fibDSWriter.removeFibEntryFromDS(rd, vrfEntry.getDestPrefix());
3093                         }
3094                     } else if (vrfTable.getMacVrfEntry() != null) {
3095                         for (MacVrfEntry macEntry : vrfTable.getMacVrfEntry()) {
3096                             if (RouteOrigin.value(macEntry.getOrigin()) != RouteOrigin.BGP) {
3097                                 //route cleanup is only meant for the routes learned through BGP.
3098                                 continue;
3099                             }
3100                             totalExternalMacRoutes++;
3101                             fibDSWriter.removeMacEntryFromDS(rd, macEntry.getMac());
3102                         }
3103                     }
3104                 }
3105             } else {
3106                 LOG.error("deleteExternalFibRoutes:: FIBentries.class is not present");
3107             }
3108         } catch (ReadFailedException e) {
3109             LOG.error("deleteExternalFibRoutes:: error ", e);
3110         }
3111         LOG.debug("deleted {} fib entries {} mac entries", totalExternalRoutes, totalExternalMacRoutes);
3112     }
3113
3114     public boolean addToRt2TepMap(String rd, String tepIp, String mac, Uint32 l2vni) {
3115         boolean isFirstMacUpdateFromTep = false;
3116         if (rt2TepMap.containsKey(rd)) {
3117             if (rt2TepMap.get(rd).containsKey(tepIp)) {
3118                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and tep-ip {}. No Elan DS write required",
3119                         mac, l2vni, rd, tepIp);
3120                 rt2TepMap.get(rd).get(tepIp).put(mac, l2vni);
3121             } else {
3122                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and new tep-ip {}",
3123                         mac, l2vni, rd, tepIp);
3124                 isFirstMacUpdateFromTep = true;
3125                 Map<String, Uint32> macList = new HashMap<>();
3126                 macList.put(mac, l2vni);
3127                 rt2TepMap.get(rd).put(tepIp, macList);
3128             }
3129         } else {
3130             LOG.debug("RT2 with mac {} l2vni {} from new rd {} and tep ip {}",
3131                     mac, l2vni, rd, tepIp);
3132             isFirstMacUpdateFromTep = true;
3133             Map<String, Uint32> macList = new HashMap<>();
3134             macList.put(mac, l2vni);
3135             Map<String, Map<String, Uint32>> tepIpMacMap = new HashMap<>();
3136             tepIpMacMap.put(tepIp, macList);
3137             rt2TepMap.put(rd, tepIpMacMap);
3138         }
3139         return isFirstMacUpdateFromTep;
3140     }
3141
3142     public boolean deleteFromRt2TepMap(String rd, String tepIp, String mac) {
3143         boolean isLastMacUpdateFromTep = false;
3144         LOG.debug("RT2 withdraw with rd {} mac {} tep-ip {} ", rd, mac, tepIp);
3145         if (rt2TepMap.containsKey(rd)) {
3146             if (rt2TepMap.get(rd).containsKey(tepIp)) {
3147                 if (rt2TepMap.get(rd).get(tepIp).containsKey(mac)) {
3148                     LOG.debug("RT2 Withdraw : Removing the mac {} from Map", mac);
3149                     rt2TepMap.get(rd).get(tepIp).remove(mac);
3150                     if (rt2TepMap.get(rd).get(tepIp).isEmpty()) {
3151                         isLastMacUpdateFromTep = true;
3152                         LOG.debug("RT2 Withdraw : Removing the tep-ip {} from Map", tepIp);
3153                         rt2TepMap.get(rd).remove(tepIp);
3154                         if (rt2TepMap.get(rd).isEmpty()) {
3155                             LOG.debug("RT2 Withdraw : Removing the rd {} from Map", rd);
3156                             rt2TepMap.remove(rd);
3157                         }
3158                     }
3159                 }
3160             }
3161         }
3162         return isLastMacUpdateFromTep;
3163     }
3164
3165     public Collection<String> getTepIPs(String rd) {
3166         final Map<String, Map<String, Uint32>> tepIpMap = rt2TepMap.get(rd);
3167         return tepIpMap != null ? tepIpMap.keySet() : Collections.emptyList();
3168     }
3169
3170     public boolean isBgpConnected() {
3171         return (bgpRouter == null) ? false : bgpRouter.isBgpConnected();
3172     }
3173
3174     public long getLastConnectedTS() {
3175         return (bgpRouter == null) ? 0 : bgpRouter.getLastConnectedTS();
3176     }
3177
3178     public long getConnectTS() {
3179         return (bgpRouter == null) ? 0 : bgpRouter.getConnectTS();
3180     }
3181
3182     public long getStartTS() {
3183         return (bgpRouter == null) ? 0 : bgpRouter.getStartTS();
3184     }
3185
3186     public TTransport getTransport() {
3187         return bgpRouter.getTransport();
3188     }
3189
3190     public int getTotalStaledCount() {
3191         return totalStaledCount;
3192     }
3193
3194     public int getTotalCleared() {
3195         return totalCleared;
3196     }
3197
3198     public BgpCounters getBgpCounters() {
3199         return bgpCountersReference.get();
3200     }
3201
3202     private void startBgpCountersTask() {
3203         if (getBgpCounters() == null && bgpCountersReference.compareAndSet(null,
3204                 new BgpCounters(getBgpSdncMipIp(), metricProvider))) {
3205             bgpCountersTask = executor.scheduleAtFixedRate(bgpCountersReference.get(), 0, 120 * 1000,
3206                     TimeUnit.MILLISECONDS);
3207             LOG.info("Bgp Counters task scheduled for every two minutes.");
3208
3209             bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
3210         }
3211     }
3212
3213     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3214             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3215     private void stopBgpCountersTask() {
3216         final BgpCounters bgpCounters = bgpCountersReference.getAndSet(null);
3217         if (bgpCounters != null) {
3218             bgpCountersTask.cancel(true);
3219             bgpCounters.close();
3220         }
3221     }
3222
3223     private void startBgpAlarmsTask() {
3224         if (getBgpAlarms() == null && bgpAlarmsReference.compareAndSet(null, new BgpAlarms(this))) {
3225             bgpAlarmsReference.get().init();
3226             bgpAlarmsTask = executor.scheduleAtFixedRate(bgpAlarmsReference.get(), 0, 60 * 1000, TimeUnit.MILLISECONDS);
3227             LOG.info("Bgp Alarms task scheduled for every minute.");
3228         } else {
3229             LOG.trace("Bgp Alarms task already scheduled for every minute.");
3230         }
3231     }
3232
3233     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3234             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3235     private void stopBgpAlarmsTask() {
3236         final BgpAlarms bgpAlarms = bgpAlarmsReference.getAndSet(null);
3237         if (bgpAlarms != null) {
3238             bgpAlarmsTask.cancel(true);
3239             bgpAlarms.close();
3240         }
3241     }
3242
3243     public BgpAlarms getBgpAlarms() {
3244         return bgpAlarmsReference.get();
3245     }
3246
3247     public void getPeerStatus(String nbrIp, long nbrAsNum) throws
3248             BgpRouterException, TException {
3249         bgpRouter.getPeerStatus(nbrIp, nbrAsNum);
3250     }
3251
3252     private static String appendNextHopToPrefix(String prefix, String nextHop) {
3253         return prefix + ":" + nextHop;
3254     }
3255
3256     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3257             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3258     private static String extractPrefix(String prefixNextHop) {
3259         return prefixNextHop.split(":")[0];
3260     }
3261
3262     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3263             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3264     private static String extractNextHop(String prefixNextHop) {
3265         return prefixNextHop.split(":")[1];
3266     }
3267
3268     private static String extractMd5Secret(final Neighbors val) {
3269         String md5Secret = null;
3270         TcpSecurityOption tcpSecOpt = val.getTcpSecurityOption();
3271         if (tcpSecOpt != null) {
3272             if (tcpSecOpt instanceof TcpMd5SignatureOption) {
3273                 md5Secret = ((TcpMd5SignatureOption) tcpSecOpt).getTcpMd5SignaturePassword().getValue();
3274             } else { // unknown TcpSecurityOption
3275                 LOG.debug("neighbors  Ignored unknown tcp-security-option of peer {}", val.getAddress().getValue());
3276             }
3277         }
3278         return md5Secret;
3279     } // private method extractMd5Secret
3280
3281     @SuppressWarnings("checkstyle:IllegalCatch")
3282     @Override
3283     public ListenableFuture<RpcResult<InitiateEorOutput>> initiateEor(InitiateEorInput input) {
3284         boolean returnError = false;
3285         String msg = null;
3286         String neighborIp = null;
3287         if (!isBGPEntityOwner()) {
3288             msg = String.format("RPC triggered in Non-EoS Owner");
3289             return Futures.immediateFuture(
3290                     RpcResultBuilder.<InitiateEorOutput>failed().withError(RpcError.ErrorType.APPLICATION,
3291                             msg).build());
3292         }
3293         if (input == null) {
3294             msg = String.format("BGP invalid input for EoR");
3295             LOG.error("Error : {}", msg);
3296             returnError = true;
3297         } else {
3298             neighborIp = input.getNeighborIp();
3299         }
3300         if (eorSupressedDuetoUpgradeFlag.get() == false) {
3301             msg = String.format("EoR triggerd by RBU-RPC call before replay"
3302                     + "of BGP configuration (or) BGP not restarted");
3303             LOG.error("Error : {}", msg);
3304         }
3305         if ("ALL".compareToIgnoreCase(neighborIp) == 0) {
3306             //send EoR for all the neighbor
3307             LOG.error("EoR trigger received to ALL neighbors");
3308             final int numberOfEORRetries = 3;
3309             RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
3310             do {
3311                 try {
3312                     BgpRouter br = bgpRouter;
3313                     br.sendEOR();
3314                     LOG.debug("RPC: sendEOR {} successful", br);
3315                     break;
3316                 } catch (Exception e) {
3317                     eorRetry.errorOccured();
3318                     LOG.error("Replay:sedEOR() received exception:", e);
3319                 }
3320             } while (eorRetry.shouldRetry());
3321             eorSupressedDuetoUpgradeFlag.set(false);
3322         } else if (InetAddresses.isInetAddress(neighborIp)) {
3323             //send EoR for only one neighbor
3324             msg = String.format("Inidividual neighbors EoR is not supported");
3325             LOG.warn("Error : {}", msg);
3326             returnError = true;
3327         } else {
3328             //error
3329             msg = String.format("RPC: initiateEor: Invalid input ");
3330             LOG.warn("Error : {}", msg);
3331             returnError = true;
3332         }
3333         if (returnError) {
3334             return Futures.immediateFuture(
3335                     RpcResultBuilder.<InitiateEorOutput>failed().withError(RpcError.ErrorType.APPLICATION,
3336                             msg).build());
3337         }
3338         InitiateEorOutput initiateEorOutput =
3339                 new InitiateEorOutputBuilder().setRetVal(0L).build();
3340         return Futures.immediateFuture(RpcResultBuilder.<InitiateEorOutput>success()
3341                 .withResult(initiateEorOutput).build());
3342     }
3343 }