bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import static org.opendaylight.netvirt.bgpmanager.oam.BgpConstants.HISTORY_LIMIT;
  11 import static org.opendaylight.netvirt.bgpmanager.oam.BgpConstants.HISTORY_THRESHOLD;
  12
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.net.InetAddresses;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.ThreadFactoryBuilder;
  18 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  19 import io.netty.util.concurrent.GlobalEventExecutor;
  20 import java.lang.reflect.Constructor;
  21 import java.lang.reflect.InvocationTargetException;
  22 import java.net.InetAddress;
  23 import java.net.InetSocketAddress;
  24 import java.net.NetworkInterface;
  25 import java.net.SocketException;
  26 import java.util.ArrayList;
  27 import java.util.Collection;
  28 import java.util.Collections;
  29 import java.util.Enumeration;
  30 import java.util.HashMap;
  31 import java.util.Iterator;
  32 import java.util.List;
  33 import java.util.Map;
  34 import java.util.Objects;
  35 import java.util.Optional;
  36 import java.util.Timer;
  37 import java.util.concurrent.Callable;
  38 import java.util.concurrent.ConcurrentHashMap;
  39 import java.util.concurrent.CountDownLatch;
  40 import java.util.concurrent.ExecutionException;
  41 import java.util.concurrent.Executors;
  42 import java.util.concurrent.Future;
  43 import java.util.concurrent.ScheduledExecutorService;
  44 import java.util.concurrent.ScheduledFuture;
  45 import java.util.concurrent.TimeUnit;
  46 import java.util.concurrent.TimeoutException;
  47 import java.util.concurrent.atomic.AtomicBoolean;
  48 import java.util.concurrent.atomic.AtomicInteger;
  49 import java.util.concurrent.atomic.AtomicReference;
  50 import javax.annotation.PreDestroy;
  51 import javax.inject.Inject;
  52 import javax.inject.Singleton;
  53 import org.apache.thrift.TApplicationException;
  54 import org.apache.thrift.TException;
  55 import org.apache.thrift.transport.TTransport;
  56 import org.eclipse.jdt.annotation.Nullable;
  57 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  60 import org.opendaylight.infrautils.metrics.MetricProvider;
  61 import org.opendaylight.mdsal.binding.api.ClusteredDataTreeChangeListener;
  62 import org.opendaylight.mdsal.binding.api.DataBroker;
  63 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  64 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
  65 import org.opendaylight.mdsal.eos.binding.api.Entity;
  66 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipCandidateRegistration;
  67 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipListenerRegistration;
  68 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  69 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  70 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  71 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  72 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  73 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  74 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  75 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  76 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  77 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  78 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  79 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  80 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  81 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  82 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  83 import org.opendaylight.netvirt.bgpmanager.thrift.gen.protocol_type;
  84 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  85 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  86 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  87 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  88 import org.opendaylight.ovsdb.utils.mdsal.utils.TransactionHistory;
  89 import org.opendaylight.ovsdb.utils.mdsal.utils.TransactionType;
  90 import org.opendaylight.serviceutils.tools.listener.AbstractAsyncDataTreeChangeListener;
  91 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebfd.rev190219.BfdConfig;
  92 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebfd.rev190219.BfdConfigBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.AddressFamily;
  94 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  95 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
  96 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EbgpService;
  97 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EncapType;
  98 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorInput;
  99 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorOutput;
 100 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorOutputBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.TcpMd5SignaturePasswordType;
 102 //import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.*;
 103 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
 104 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
 106 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.DcgwTepList;
 108 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
 109 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
 110 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
 111 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathContainer;
 113 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsContainer;
 114 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksContainer;
 115 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathContainer;
 116 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsContainer;
 117 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTep;
 118 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTepBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTepKey;
 120 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.Multipath;
 121 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.MultipathBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.MultipathKey;
 123 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.Neighbors;
 124 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.NeighborsBuilder;
 125 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.NeighborsKey;
 126 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamilies;
 127 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamiliesBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamiliesKey;
 129 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.EbgpMultihop;
 130 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.EbgpMultihopBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.UpdateSource;
 132 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.UpdateSourceBuilder;
 133 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.Networks;
 134 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.NetworksBuilder;
 135 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.NetworksKey;
 136 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpath;
 137 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpathBuilder;
 138 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpathKey;
 139 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.Vrfs;
 140 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.VrfsBuilder;
 141 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.VrfsKey;
 142 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.vrfs.AddressFamiliesVrf;
 143 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.vrfs.AddressFamiliesVrfBuilder;
 144 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.TcpSecurityOption;
 145 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOption;
 146 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOptionBuilder;
 147 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
 148 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
 149 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
 150 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 151 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 152 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.macvrfentries.MacVrfEntry;
 153 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 154 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 155 import org.opendaylight.yangtools.yang.binding.DataObject;
 156 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 157 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 158 import org.opendaylight.yangtools.yang.common.RpcError;
 159 import org.opendaylight.yangtools.yang.common.RpcResult;
 160 import org.opendaylight.yangtools.yang.common.RpcResultBuilder;
 161 import org.opendaylight.yangtools.yang.common.Uint32;
 162 import org.osgi.framework.BundleContext;
 163 import org.osgi.util.tracker.ServiceTracker;
 164 import org.slf4j.Logger;
 165 import org.slf4j.LoggerFactory;
 166
 167 @Singleton
 168 public class BgpConfigurationManager implements EbgpService {
 169     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 170
 171     // to have stale FIB map (RD, Prefix)
 172     //  number of seconds wait for route sync-up between ODL and BGP
 173     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 174     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 175     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 176     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 177     private static final String DEF_UPORT = "6644";
 178     private static final String DEF_CHOST = "255.255.255.255"; // Invalid Host IP
 179     private static final String DEF_CPORT = "7644";
 180     private static final String DEF_BGP_SDNC_MIP = "127.0.0.1";
 181     //vpnservice.bgp.thrift.bgp.mip is the MIP present with ODL. Here we open 6644 port
 182     private static final String BGP_SDNC_MIP = "vpnservice.bgp.thrift.bgp.mip";
 183     private static final String BGP_GR_RESTART_TIMER_PROPERTY = "vpnservice.bgp.gr.timer";
 184     private static final String BGP_KA_TIMER_PROPERTY = "vpnservice.bgp.ka.timer";
 185     private static final String BGP_HOLD_TIMER_PROPERTY = "vpnservice.bgp.hold.timer";
 186     private static final String BGP_EOR_DELAY_PROPERTY = "vpnservice.bgp.eordelay";
 187     private static final int DEF_BGP_KA_TIME = 60;
 188     private static final int DEF_BGP_HOLD_TIME = 180;
 189     private static final int DEF_BGP_GR_TIME = 4000;
 190     private static final int RESTART_DEFAULT_GR = 90;
 191     private static final int DS_RETRY_COUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
 192     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
 193     private static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 194     private static final String BGP_EOR_DELAY = "vpnservice.bgp.eordelay";
 195     private static final String DEF_BGP_EOR_DELAY = "1800";
 196     private static final String BGP_ENTITY_NAME = "bgp";
 197     private static final String ADD_WARN = "Config store updated; undo with Delete if needed.";
 198     private static final String DEL_WARN = "Config store updated; undo with Add if needed.";
 199     private static final String UPD_WARN = "Update operation not supported; Config store updated;"
 200             + " restore with another Update if needed.";
 201     private static long bgp_as_num = 0;
 202     private static List<Neighbors> nbrList = new ArrayList<>();
 203     private int bgpKaTime = 0;
 204     private int bgpHoldTime = 0;
 205     private int bgpGrRestartTime = 0;
 206
 207     private static final Class<?>[] REACTORS = {
 208         ConfigServerReactor.class, AsIdReactor.class,
 209         GracefulRestartReactor.class, LoggingReactor.class,
 210         NeighborsReactor.class, UpdateSourceReactor.class,
 211         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 212         NetworksReactor.class, VrfsReactor.class, BgpReactor.class,
 213         MultipathReactor.class, VrfMaxpathReactor.class, BfdConfigReactor.class
 214     };
 215
 216     private IBgpManager bgpManager;
 217     private final DataBroker dataBroker;
 218     private final FibDSWriter fibDSWriter;
 219     private final IVpnLinkService vpnLinkService;
 220     private final BundleContext bundleContext;
 221     private final BgpUtil bgpUtil;
 222     private volatile Bgp config;
 223     private final BgpRouter bgpRouter;
 224     private final BgpSyncHandle bgpSyncHandle = new BgpSyncHandle();
 225     private volatile BgpThriftService bgpThriftService = null;
 226     private final int delayEorSeconds;
 227
 228     private final CountDownLatch initer = new CountDownLatch(1);
 229
 230     private final String hostStartup;
 231     private final String portStartup;
 232
 233     private final AtomicReference<BgpCounters> bgpCountersReference = new AtomicReference<>();
 234     private ScheduledFuture<?> bgpCountersTask;
 235
 236     private final AtomicReference<BgpAlarms> bgpAlarmsReference = new AtomicReference<>();
 237     private ScheduledFuture<?> bgpAlarmsTask;
 238
 239     private Future<?> lastReplayJobFt;
 240     private ScheduledFuture<?> routeCleanupFuture;
 241
 242     private long staleStartTime;
 243     private long staleEndTime;
 244     private long cfgReplayStartTime;
 245     private long cfgReplayEndTime;
 246     private long staleCleanupTime;
 247     private int totalStaledCount;
 248     private int totalCleared;
 249     private int totalExternalRoutes;
 250     private int totalExternalMacRoutes;
 251
 252     private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(
 253             new ThreadFactoryBuilder().setNameFormat("bgp-config-%d").setDaemon(true).build());
 254
 255     /**
 256      * this map store the new address families to send to quagga. When it is sended you must clear it.
 257      * The keys String are rd (route distinguisher).
 258      * */
 259     private final ConcurrentHashMap<String, List<AddressFamiliesVrf>> mapNewAdFamily = new ConcurrentHashMap<>();
 260
 261     // map<rd, map<prefix/len:nexthop, label>>
 262     private final Map<String, Map<String, Uint32>> staledFibEntriesMap = new ConcurrentHashMap<>();
 263
 264     // map<rd, map<tep-ip, map<mac, l2vni>>>
 265     private final Map<String, Map<String, Map<String, Uint32>>> rt2TepMap = new ConcurrentHashMap<>();
 266
 267     //map<rd+prefix/plen, list (nexthop)>
 268     private final Map<String,List> fibMap = new HashMap<>();
 269
 270     private final List<AutoCloseable> listeners = new ArrayList<>();
 271
 272     private final EntityOwnershipUtils entityOwnershipUtils;
 273     private final EntityOwnershipCandidateRegistration candidateRegistration;
 274     private final EntityOwnershipListenerRegistration entityListenerRegistration;
 275     private final MetricProvider metricProvider;
 276     private final TransactionHistory bgpUpdatesHistory;
 277
 278     private volatile AtomicBoolean eorSupressedDuetoUpgradeFlag = new AtomicBoolean(false);
 279
 280     @Inject
 281     public BgpConfigurationManager(final DataBroker dataBroker,
 282             final EntityOwnershipService entityOwnershipService,
 283             final FibDSWriter fibDSWriter,
 284             final IVpnLinkService vpnLinkSrvce,
 285             final BundleContext bundleContext,
 286             final BgpUtil bgpUtil,
 287             final MetricProvider metricProvider) {
 288         this.dataBroker = dataBroker;
 289         this.fibDSWriter = fibDSWriter;
 290         this.vpnLinkService = vpnLinkSrvce;
 291         this.bundleContext = bundleContext;
 292         this.bgpUtil = bgpUtil;
 293         this.metricProvider = metricProvider;
 294         hostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 295         portStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 296         bgpKaTime =
 297                 Integer.parseInt(getProperty(BGP_KA_TIMER_PROPERTY,
 298                         Integer.toString(DEF_BGP_KA_TIME)));
 299         bgpHoldTime =
 300                 Integer.parseInt(getProperty(BGP_HOLD_TIMER_PROPERTY,
 301                         Integer.toString(DEF_BGP_HOLD_TIME)));
 302         bgpGrRestartTime =
 303                 Integer.parseInt(getProperty(BGP_GR_RESTART_TIMER_PROPERTY,
 304                         Integer.toString(DEF_BGP_GR_TIME)));
 305         LOG.info("ConfigServer at {}:{}", hostStartup, portStartup);
 306         VtyshCli.setHostAddr(hostStartup);
 307         ClearBgpCli.setHostAddr(hostStartup);
 308         bgpUpdatesHistory = new TransactionHistory(HISTORY_LIMIT, HISTORY_THRESHOLD);
 309         bgpRouter = BgpRouter.newInstance(this::getConfig, this::isBGPEntityOwner, bgpUpdatesHistory);
 310         delayEorSeconds = Integer.parseInt(getProperty(BGP_EOR_DELAY, DEF_BGP_EOR_DELAY));
 311
 312         entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 313
 314         candidateRegistration = registerEntityCandidate(entityOwnershipService);
 315         entityListenerRegistration = registerEntityListener(entityOwnershipService);
 316
 317         LOG.info("BGP Configuration manager initialized");
 318         initer.countDown();
 319
 320         GlobalEventExecutor.INSTANCE.execute(() -> {
 321             ServiceTracker<IBgpManager, ?> tracker = null;
 322             try {
 323                 tracker = new ServiceTracker<>(bundleContext, IBgpManager.class, null);
 324                 tracker.open();
 325                 bgpManager = (IBgpManager) tracker.waitForService(TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES));
 326                 Preconditions.checkState(bgpManager != null, "IBgpManager service not found");
 327             } catch (InterruptedException e) {
 328                 throw new IllegalStateException("Error retrieving IBgpManager service", e);
 329             } finally {
 330                 if (tracker != null) {
 331                     tracker.close();
 332                 }
 333             }
 334
 335             String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
 336             if (InetAddresses.isInetAddress(getBgpSdncMipIp())) {
 337                 InetSocketAddress bgpThriftServerSocketAddr = new InetSocketAddress(getBgpSdncMipIp(),
 338                         Integer.parseInt(updatePort));
 339                 bgpThriftService = new BgpThriftService(bgpThriftServerSocketAddr, bgpManager, this);
 340                 if (isBGPEntityOwner()) {
 341                     //I am EoS owner of BGP, opening bgp thrift UPDATE-SERVER port.
 342                     LOG.info("BGP Configuration manager initialized: UPDATE-SERVER started");
 343                     bgpThriftService.start();
 344                 }
 345                 LOG.info("UPDATE server started :ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 346             } else {
 347                 LOG.error("Failed to init UPDATE server invalid ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 348             }
 349         });
 350         registerCallbacks();
 351         LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 352     }
 353
 354     public String getBgpSdncMipIp() {
 355         return getProperty(BGP_SDNC_MIP, DEF_BGP_SDNC_MIP);
 356     }
 357
 358     public long getStaleCleanupTime() {
 359         return staleCleanupTime;
 360     }
 361
 362     public void setStaleCleanupTime(long staleCleanupTime) {
 363         this.staleCleanupTime = staleCleanupTime;
 364     }
 365
 366     public long getCfgReplayEndTime() {
 367         return cfgReplayEndTime;
 368     }
 369
 370     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 371         this.cfgReplayEndTime = cfgReplayEndTime;
 372     }
 373
 374     public TransactionHistory getBgpUpdatesHistory() {
 375         return bgpUpdatesHistory;
 376     }
 377
 378     public long getCfgReplayStartTime() {
 379         return cfgReplayStartTime;
 380     }
 381
 382     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 383         this.cfgReplayStartTime = cfgReplayStartTime;
 384     }
 385
 386     public long getStaleEndTime() {
 387         return staleEndTime;
 388     }
 389
 390     public void setStaleEndTime(long staleEndTime) {
 391         this.staleEndTime = staleEndTime;
 392     }
 393
 394     public long getStaleStartTime() {
 395         return staleStartTime;
 396     }
 397
 398     public void setStaleStartTime(long staleStartTime) {
 399         this.staleStartTime = staleStartTime;
 400     }
 401
 402     private Object createListener(Class<?> cls) {
 403         try {
 404             Constructor<?> ctor = cls.getConstructor(BgpConfigurationManager.class);
 405             return ctor.newInstance(this);
 406         } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException
 407                 e) {
 408             LOG.error("Failed to create listener object", e);
 409             return null;
 410         }
 411     }
 412
 413     private void registerCallbacks() {
 414         for (Class<?> reactor : REACTORS) {
 415             Object obj = createListener(reactor);
 416             if (obj != null) {
 417                 AbstractAsyncDataTreeChangeListener dcl = (AbstractAsyncDataTreeChangeListener) obj;
 418                 dcl.register();
 419                 listeners.add(dcl);
 420             }
 421         }
 422     }
 423
 424     @SuppressWarnings("checkstyle:IllegalCatch")
 425     @PreDestroy
 426     public void close() {
 427         executor.shutdown();
 428
 429         if (bgpThriftService != null) {
 430             bgpThriftService.stop();
 431             bgpThriftService = null;
 432         }
 433
 434         if (isBgpConnected()) {
 435             //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 436             bgpRouter.disconnect();
 437         }
 438
 439         if (candidateRegistration != null) {
 440             candidateRegistration.close();
 441         }
 442
 443         entityListenerRegistration.close();
 444
 445         listeners.forEach(l -> {
 446             try {
 447                 l.close();
 448             } catch (Exception e) {
 449                 LOG.warn("Error closing {}", l ,e);
 450             }
 451         });
 452
 453         LOG.info("{} close", getClass().getSimpleName());
 454     }
 455
 456     private String getProperty(String var, String def) {
 457         String property = bundleContext.getProperty(var);
 458         return property == null ? def : property;
 459     }
 460
 461     private EntityOwnershipCandidateRegistration registerEntityCandidate(
 462             final EntityOwnershipService entityOwnershipService) {
 463         try {
 464             return entityOwnershipService.registerCandidate(
 465                     new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME));
 466         } catch (CandidateAlreadyRegisteredException e) {
 467             LOG.error("failed to register bgp entity", e);
 468             return null;
 469         }
 470     }
 471
 472     private EntityOwnershipListenerRegistration registerEntityListener(
 473             final EntityOwnershipService entityOwnershipService) {
 474         return entityOwnershipService.registerListener(BGP_ENTITY_TYPE_FOR_OWNERSHIP, ownershipChange -> {
 475             LOG.info("entity owner change event fired: {}", ownershipChange);
 476
 477             if (ownershipChange.getState() == EntityOwnershipChangeState.LOCAL_OWNERSHIP_GRANTED) {
 478                 LOG.info("This PL is the Owner");
 479                 if (bgpThriftService != null) {
 480                     //opening UPDATE-SERVER port.
 481                     bgpThriftService.start();
 482                 } else {
 483                     LOG.error("I am the owner of BGP entity, but bgpThriftService is not initialized yet");
 484                 }
 485                 bgpRestarted();
 486             } else {
 487                 LOG.info("Not owner: hasOwner: {}, isOwner: {}", ownershipChange.getState().hasOwner(),
 488                         ownershipChange.getState().isOwner());
 489                 if (bgpThriftService != null && bgpThriftService.isBgpThriftServiceStarted()) {
 490                     //close the bgp Thrift Update-SERVER port opened on non-Entity Owner
 491                     bgpThriftService.stop();
 492                 }
 493                 if (isBgpConnected()) {
 494                     //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 495                     bgpRouter.disconnect();
 496                 }
 497                 stopBgpCountersTask();
 498                 stopBgpAlarmsTask();
 499             }
 500         });
 501     }
 502
 503     public boolean isBGPEntityOwner() {
 504         if (entityOwnershipUtils == null) {
 505             LOG.error("entityOwnershipUtils is NULL when listener callbacks fired");
 506             return false;
 507         }
 508         return entityOwnershipUtils.isEntityOwner(new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME), 0, 1);
 509     }
 510
 511     public Bgp get() {
 512         config = getConfig();
 513         return config;
 514     }
 515
 516     public class ConfigServerReactor
 517             extends AbstractAsyncDataTreeChangeListener<ConfigServer>
 518             implements ClusteredDataTreeChangeListener<ConfigServer> {
 519         private static final String YANG_OBJ = "config-server ";
 520
 521         public ConfigServerReactor() {
 522             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 523                     InstanceIdentifier.create(Bgp.class).child(ConfigServer.class),
 524                     org.opendaylight.infrautils.utils.concurrent.Executors
 525                             .newListeningSingleThreadExecutor("ConfigServerReactor", LOG));
 526         }
 527
 528         @Override
 529         public void add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 530             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 531             if (!isBGPEntityOwner()) {
 532                 return;
 533             }
 534
 535             try {
 536                 initer.await();
 537             } catch (InterruptedException e) {
 538                 // Ignored
 539             }
 540             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 541             bgpRouter.configServerUpdated();
 542             synchronized (BgpConfigurationManager.this) {
 543                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 544                         val.getPort().intValue());
 545                 if (!res) {
 546                     LOG.error(YANG_OBJ + "Add failed; " + ADD_WARN);
 547                 }
 548             }
 549             VtyshCli.setHostAddr(val.getHost().getValue());
 550             ClearBgpCli.setHostAddr(val.getHost().getValue());
 551         }
 552
 553         @Override
 554         public void remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 555             LOG.trace("received bgp disconnect");
 556             if (!isBGPEntityOwner()) {
 557                 return;
 558             }
 559
 560             bgpRouter.configServerUpdated();
 561
 562             synchronized (BgpConfigurationManager.this) {
 563                 if (bgp_as_num != 0) {
 564                     try {
 565                         bgpRouter.stopBgp(bgp_as_num);
 566                         stopBgpCountersTask();
 567                         stopBgpAlarmsTask();
 568                     } catch (TException | BgpRouterException e) {
 569                         LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 570                     }
 571                 } else {
 572                     LOG.debug("bgp as-id is null while removing config-server");
 573                 }
 574                 bgpRouter.disconnect();
 575             }
 576         }
 577
 578         @Override
 579         public void update(InstanceIdentifier<ConfigServer> iid,
 580                 ConfigServer oldval, ConfigServer newval) {
 581             LOG.trace("received bgp Connection update");
 582             if (!isBGPEntityOwner()) {
 583                 return;
 584             }
 585             LOG.error(YANG_OBJ + UPD_WARN);
 586         }
 587     }
 588
 589     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
 590             justification = "https://github.com/spotbugs/spotbugs/issues/811")
 591     private BgpRouter getClient(String yangObj) {
 592         if (bgpRouter == null || !bgpRouter.isBgpConnected()) {
 593             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 594             return null;
 595         }
 596         return bgpRouter;
 597     }
 598
 599     public class AsIdReactor extends AbstractAsyncDataTreeChangeListener<AsId>
 600             implements ClusteredDataTreeChangeListener<AsId> {
 601
 602         private static final String YANG_OBJ = "as-id ";
 603
 604         public AsIdReactor() {
 605             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 606                     InstanceIdentifier.create(Bgp.class).child(AsId.class),
 607                     org.opendaylight.infrautils.utils.concurrent.Executors
 608                             .newListeningSingleThreadExecutor("AsIdReactor", LOG));
 609         }
 610
 611         @SuppressFBWarnings(value  = "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD")
 612         @Override
 613         public void add(InstanceIdentifier<AsId> iid, AsId val) {
 614             LOG.error("received bgp add asid {}", val);
 615             if (!isBGPEntityOwner()) {
 616                 return;
 617             }
 618             LOG.debug("received add router config asNum {}", val.getLocalAs());
 619             bgp_as_num = val.getLocalAs().longValue();
 620             synchronized (BgpConfigurationManager.this) {
 621                 BgpRouter br = getClient(YANG_OBJ);
 622                 if (br == null) {
 623                     LOG.debug("{} Unable to process add for asNum {}; {} {}", YANG_OBJ, val.getLocalAs(),
 624                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 625                     return;
 626                 }
 627                 bgpRestarted();
 628
 629                 startBgpCountersTask();
 630                 startBgpAlarmsTask();
 631             }
 632         }
 633
 634         @Override
 635         public void remove(InstanceIdentifier<AsId> iid, AsId val) {
 636             LOG.error("received delete router config asNum {}", val.getLocalAs());
 637             if (!isBGPEntityOwner()) {
 638                 return;
 639             }
 640             synchronized (BgpConfigurationManager.this) {
 641                 long asNum = val.getLocalAs().toJava();
 642                 BgpRouter br = getClient(YANG_OBJ);
 643                 bgp_as_num = 0;
 644                 if (br == null) {
 645                     LOG.debug("{} Unable to process remove for asNum {}; {} {}", YANG_OBJ, asNum,
 646                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 647                     return;
 648                 }
 649                 try {
 650                     br.stopBgp(asNum);
 651                 } catch (TException | BgpRouterException e) {
 652                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 653                 }
 654
 655                 stopBgpCountersTask();
 656                 stopBgpAlarmsTask();
 657
 658                 Bgp conf = getConfig();
 659                 if (conf == null) {
 660                     LOG.error("Config Null while removing the as-id");
 661                     return;
 662                 }
 663                 LOG.debug("Removing external routes from FIB");
 664                 deleteExternalFibRoutes();
 665                 List<Neighbors> nbrs = conf.getNeighborsContainer() == null ? null
 666                         : conf.getNeighborsContainer().getNeighbors();
 667                 if (nbrs != null && nbrs.size() > 0) {
 668                     LOG.error("Tring to remove the as-id when neighbor config is already present");
 669                     for (Neighbors nbr : nbrs) {
 670                         LOG.debug("Removing Neighbor {} from Data store", nbr.getAddress().getValue());
 671                         delNeighbor(nbr.getAddress().getValue());
 672                     }
 673                 }
 674             }
 675         }
 676
 677         @Override
 678         public void update(InstanceIdentifier<AsId> iid,
 679                 AsId oldval, AsId newval) {
 680             if (!isBGPEntityOwner()) {
 681                 return;
 682             }
 683             LOG.error(YANG_OBJ + UPD_WARN);
 684         }
 685     }
 686
 687     public class GracefulRestartReactor extends AbstractAsyncDataTreeChangeListener<GracefulRestart>
 688             implements ClusteredDataTreeChangeListener<GracefulRestart> {
 689
 690         private static final String YANG_OBJ = "graceful-restart ";
 691
 692         public GracefulRestartReactor() {
 693             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 694                     InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class),
 695                     org.opendaylight.infrautils.utils.concurrent.Executors
 696                             .newListeningSingleThreadExecutor("GracefulRestartReactor", LOG));
 697         }
 698
 699         @Override
 700         public void add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 701             if (!isBGPEntityOwner()) {
 702                 return;
 703             }
 704             synchronized (BgpConfigurationManager.this) {
 705                 int stalePathTime = val.getStalepathTime().intValue();
 706                 BgpRouter br = getClient(YANG_OBJ);
 707                 if (br == null) {
 708                     LOG.error("{} Unable to add stale-path time {}; {} {}", YANG_OBJ, stalePathTime,
 709                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 710                     return;
 711                 }
 712                 try {
 713                     br.addGracefulRestart(stalePathTime);
 714                 } catch (TException | BgpRouterException e) {
 715                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 716                 }
 717             }
 718         }
 719
 720         @Override
 721         public void remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 722             if (!isBGPEntityOwner()) {
 723                 return;
 724             }
 725             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 726             synchronized (BgpConfigurationManager.this) {
 727                 BgpRouter br = getClient(YANG_OBJ);
 728                 if (br == null) {
 729                     LOG.error("{} Unable to delete stale-path time; {} {}", YANG_OBJ,
 730                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 731                     return;
 732                 }
 733                 try {
 734                     br.delGracefulRestart();
 735                 } catch (TException | BgpRouterException e) {
 736                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 737                 }
 738             }
 739         }
 740
 741         @Override
 742         public void update(InstanceIdentifier<GracefulRestart> iid,
 743                 GracefulRestart oldval, GracefulRestart newval) {
 744             if (!isBGPEntityOwner()) {
 745                 return;
 746             }
 747             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 748             synchronized (BgpConfigurationManager.this) {
 749                 int stalePathTime = newval.getStalepathTime().intValue();
 750                 BgpRouter br = getClient(YANG_OBJ);
 751                 if (br == null) {
 752                     LOG.error("{} Unable to update stale-path time to {}; {} {}", YANG_OBJ, stalePathTime,
 753                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 754                     return;
 755                 }
 756                 try {
 757                     br.addGracefulRestart(stalePathTime);
 758                 } catch (TException | BgpRouterException e) {
 759                     LOG.error("{} update received exception; {}", YANG_OBJ, ADD_WARN, e);
 760                 }
 761             }
 762         }
 763     }
 764
 765     public class LoggingReactor extends AbstractAsyncDataTreeChangeListener<Logging>
 766             implements ClusteredDataTreeChangeListener<Logging> {
 767
 768         private static final String YANG_OBJ = "logging ";
 769
 770         public LoggingReactor() {
 771             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 772                     InstanceIdentifier.create(Bgp.class).child(Logging.class),
 773                     org.opendaylight.infrautils.utils.concurrent.Executors
 774                             .newListeningSingleThreadExecutor("LoggingReactor", LOG));
 775         }
 776
 777         @Override
 778         public void add(InstanceIdentifier<Logging> iid, Logging val) {
 779             if (!isBGPEntityOwner()) {
 780                 return;
 781             }
 782             synchronized (BgpConfigurationManager.this) {
 783                 BgpRouter br = getClient(YANG_OBJ);
 784                 if (br == null) {
 785                     LOG.error("{} Unable to add logging for qbgp; {} {}", YANG_OBJ,
 786                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 787                     return;
 788                 }
 789                 try {
 790                     br.setLogging(val.getFile(), val.getLevel());
 791                 } catch (TException | BgpRouterException e) {
 792                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 793                 }
 794             }
 795         }
 796
 797         @Override
 798         public void remove(InstanceIdentifier<Logging> iid, Logging val) {
 799             if (!isBGPEntityOwner()) {
 800                 return;
 801             }
 802             LOG.debug("received remove Logging config val {}", val.getLevel());
 803             synchronized (BgpConfigurationManager.this) {
 804                 BgpRouter br = getClient(YANG_OBJ);
 805                 if (br == null) {
 806                     LOG.error("{} Unable to remove logging for qbgp; {} {}", YANG_OBJ,
 807                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 808                     return;
 809                 }
 810                 try {
 811                     br.setLogging(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
 812                 } catch (TException | BgpRouterException e) {
 813                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 814                 }
 815             }
 816         }
 817
 818         @Override
 819         public void update(InstanceIdentifier<Logging> iid,
 820                 Logging oldval, Logging newval) {
 821             if (!isBGPEntityOwner()) {
 822                 return;
 823             }
 824             synchronized (BgpConfigurationManager.this) {
 825                 BgpRouter br = getClient(YANG_OBJ);
 826                 if (br == null) {
 827                     LOG.error("{} Unable to update logging for qbgp; {} {}", YANG_OBJ,
 828                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 829                     return;
 830                 }
 831                 try {
 832                     br.setLogging(newval.getFile(), newval.getLevel());
 833                 } catch (TException | BgpRouterException e) {
 834                     LOG.error("{} newval received exception; {}", YANG_OBJ, ADD_WARN, e);
 835                 }
 836             }
 837         }
 838     }
 839
 840     public class NeighborsReactor extends AbstractAsyncDataTreeChangeListener<Neighbors>
 841             implements ClusteredDataTreeChangeListener<Neighbors> {
 842
 843         private static final String YANG_OBJ = "neighbors ";
 844
 845         public NeighborsReactor() {
 846             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 847                     InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class),
 848                     org.opendaylight.infrautils.utils.concurrent.Executors
 849                             .newListeningSingleThreadExecutor("NeighborsReactor", LOG));
 850         }
 851
 852         @Override
 853         public void add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 854             if (nbrList != null && !nbrList.contains(val)) {
 855                 LOG.trace("Adding nbr {} to nbrlist", val.getAddress().getValue());
 856                 nbrList.add(val);
 857             }
 858             if (!isBGPEntityOwner()) {
 859                 return;
 860             }
 861             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 862             synchronized (BgpConfigurationManager.this) {
 863                 String peerIp = val.getAddress().getValue();
 864                 String sourceIp = (val.getUpdateSource() == null) ? null :
 865                                     val.getUpdateSource().getSourceIp().getValue();
 866                 int nhops = (val.getEbgpMultihop() == null) ? 0 :
 867                                     val.getEbgpMultihop().getNhops().intValue();
 868                 List<AddressFamilies> afs = val.getAddressFamilies();
 869                 long as = val.getRemoteAs().toJava();
 870                 final String md5Secret = extractMd5Secret(val);
 871                 BgpRouter br = getClient(YANG_OBJ);
 872                 if (br == null) {
 873                     LOG.debug("{} Unable to process add for peer {} as {}; {} {}", YANG_OBJ, peerIp, as,
 874                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 875                     return;
 876                 }
 877                 try {
 878                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 879                     br.addNeighbor(peerIp, as, md5Secret);
 880                     if (nhops != 0) {
 881                         br.addEbgpMultihop(peerIp, nhops);
 882                     }
 883                     if (sourceIp != null) {
 884                         br.addUpdateSource(peerIp, sourceIp);
 885                     }
 886                     if (afs != null) {
 887                         for (AddressFamilies af : afs) {
 888                             af_afi afi = af_afi.findByValue(af.getAfi().intValue());
 889                             af_safi safi = af_safi.findByValue(af.getSafi().intValue());
 890                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
 891                         }
 892                     }
 893
 894                 } catch (TException | BgpRouterException e) {
 895                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 896                 }
 897             }
 898         }
 899
 900         @Override
 901         public void remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 902             if (nbrList != null && nbrList.contains(val)) {
 903                 LOG.trace("Removing nbr {} from nbr list", val.getAddress().getValue());
 904                 nbrList.remove(val);
 905             }
 906             if (!isBGPEntityOwner()) {
 907                 return;
 908             }
 909             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 910             synchronized (BgpConfigurationManager.this) {
 911                 String peerIp = val.getAddress().getValue();
 912                 BgpRouter br = getClient(YANG_OBJ);
 913                 if (br == null) {
 914                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 915                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 916                     return;
 917                 }
 918                 try {
 919                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 920                     br.delNeighbor(peerIp);
 921                 } catch (TException | BgpRouterException e) {
 922                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 923                 }
 924
 925                 final BgpAlarms bgpAlarms = getBgpAlarms();
 926                 if (bgpAlarms != null) {
 927                     bgpAlarms.clearBgpNbrDownAlarm(peerIp);
 928                 }
 929
 930                 if (bgpUtil.isBfdEnabled()) {
 931                     final BgpCounters bgpCounters = getBgpCounters();
 932                     if (bgpCounters != null) {
 933                         bgpCounters.clearBfdNbrCounters(peerIp);
 934                     }
 935                 }
 936             }
 937         }
 938
 939         @Override
 940         public void update(InstanceIdentifier<Neighbors> iid,
 941                 Neighbors oldval, Neighbors newval) {
 942             if (!isBGPEntityOwner()) {
 943                 return;
 944             }
 945             //purposefully nothing to do.
 946         }
 947     }
 948
 949     public class EbgpMultihopReactor extends AbstractAsyncDataTreeChangeListener<EbgpMultihop>
 950             implements ClusteredDataTreeChangeListener<EbgpMultihop> {
 951
 952         private static final String YANG_OBJ = "ebgp-multihop ";
 953
 954         public EbgpMultihopReactor() {
 955             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 956                     InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
 957                     .child(EbgpMultihop.class), org.opendaylight.infrautils.utils.concurrent.Executors
 958                             .newListeningSingleThreadExecutor("EbgpMultihopReactor", LOG));
 959         }
 960
 961         @Override
 962         public void add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 963             if (!isBGPEntityOwner()) {
 964                 return;
 965             }
 966             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 967             synchronized (BgpConfigurationManager.this) {
 968                 String peerIp = val.getPeerIp().getValue();
 969                 BgpRouter br = getClient(YANG_OBJ);
 970                 if (br == null) {
 971                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
 972                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 973                     return;
 974                 }
 975                 try {
 976                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 977                 } catch (TException | BgpRouterException e) {
 978                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 979                 }
 980             }
 981         }
 982
 983         @Override
 984         public void remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 985             if (!isBGPEntityOwner()) {
 986                 return;
 987             }
 988             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 989             synchronized (BgpConfigurationManager.this) {
 990                 String peerIp = val.getPeerIp().getValue();
 991                 BgpRouter br = getClient(YANG_OBJ);
 992                 if (br == null) {
 993                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 994                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 995                     return;
 996                 }
 997                 try {
 998                     br.delEbgpMultihop(peerIp);
 999                 } catch (TException | BgpRouterException e) {
1000                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1001                 }
1002             }
1003         }
1004
1005         @Override
1006         public void update(InstanceIdentifier<EbgpMultihop> iid,
1007                 EbgpMultihop oldval, EbgpMultihop newval) {
1008             if (!isBGPEntityOwner()) {
1009                 return;
1010             }
1011             LOG.error(YANG_OBJ + UPD_WARN);
1012         }
1013     }
1014
1015     public class UpdateSourceReactor extends AbstractAsyncDataTreeChangeListener<UpdateSource>
1016             implements ClusteredDataTreeChangeListener<UpdateSource> {
1017
1018         private static final String YANG_OBJ = "update-source ";
1019
1020         public UpdateSourceReactor() {
1021             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
1022                     InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
1023                     .child(UpdateSource.class), org.opendaylight.infrautils.utils.concurrent.Executors
1024                             .newListeningSingleThreadExecutor("UpdateSourceReactor", LOG));
1025         }
1026
1027         @Override
1028         public void add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
1029             if (!isBGPEntityOwner()) {
1030                 return;
1031             }
1032             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
1033             synchronized (BgpConfigurationManager.this) {
1034                 String peerIp = val.getPeerIp().getValue();
1035                 BgpRouter br = getClient(YANG_OBJ);
1036                 if (br == null) {
1037                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1038                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1039                     return;
1040                 }
1041                 try {
1042                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
1043                 } catch (TException | BgpRouterException e) {
1044                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1045                 }
1046             }
1047         }
1048
1049         @Override
1050         public void remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
1051             if (!isBGPEntityOwner()) {
1052                 return;
1053             }
1054             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
1055             synchronized (BgpConfigurationManager.this) {
1056                 String peerIp = val.getPeerIp().getValue();
1057                 BgpRouter br = getClient(YANG_OBJ);
1058                 if (br == null) {
1059                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1060                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1061                     return;
1062                 }
1063                 try {
1064                     br.delUpdateSource(peerIp);
1065                 } catch (TException | BgpRouterException e) {
1066                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1067                 }
1068             }
1069         }
1070
1071         @Override
1072         public void update(InstanceIdentifier<UpdateSource> iid,
1073                 UpdateSource oldval, UpdateSource newval) {
1074             if (!isBGPEntityOwner()) {
1075                 return;
1076             }
1077             LOG.error(YANG_OBJ + UPD_WARN);
1078         }
1079     }
1080
1081     public class AddressFamiliesReactor extends AbstractAsyncDataTreeChangeListener<AddressFamilies>
1082             implements ClusteredDataTreeChangeListener<AddressFamilies> {
1083
1084         private static final String YANG_OBJ = "address-families ";
1085
1086         public AddressFamiliesReactor() {
1087             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
1088                     InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
1089                     .child(AddressFamilies.class), org.opendaylight.infrautils.utils.concurrent.Executors
1090                             .newListeningSingleThreadExecutor("AddressFamiliesReactor", LOG));
1091         }
1092
1093         @Override
1094         public void add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1095             if (!isBGPEntityOwner()) {
1096                 return;
1097             }
1098             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
1099             synchronized (BgpConfigurationManager.this) {
1100                 String peerIp = val.getPeerIp().getValue();
1101                 BgpRouter br = getClient(YANG_OBJ);
1102                 if (br == null) {
1103                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1104                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1105                     return;
1106                 }
1107                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1108                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1109                 try {
1110                     br.addAddressFamily(peerIp, afi, safi);
1111                 } catch (TException | BgpRouterException e) {
1112                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1113                 }
1114             }
1115         }
1116
1117         @Override
1118         public void remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1119             if (!isBGPEntityOwner()) {
1120                 return;
1121             }
1122             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
1123             synchronized (BgpConfigurationManager.this) {
1124                 String peerIp = val.getPeerIp().getValue();
1125                 BgpRouter br = getClient(YANG_OBJ);
1126                 if (br == null) {
1127                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1128                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1129                     return;
1130                 }
1131                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1132                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1133                 try {
1134                     br.delAddressFamily(peerIp, afi, safi);
1135                 } catch (TException | BgpRouterException e) {
1136                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1137                 }
1138             }
1139         }
1140
1141         @Override
1142         public void update(InstanceIdentifier<AddressFamilies> iid,
1143                 AddressFamilies oldval, AddressFamilies newval) {
1144             if (!isBGPEntityOwner()) {
1145                 return;
1146             }
1147             LOG.error(YANG_OBJ + UPD_WARN);
1148         }
1149     }
1150
1151     public class NetworksReactor extends AbstractAsyncDataTreeChangeListener<Networks>
1152             implements ClusteredDataTreeChangeListener<Networks> {
1153
1154         private static final String YANG_OBJ = "networks ";
1155
1156         public NetworksReactor() {
1157             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
1158                     InstanceIdentifier.create(Bgp.class).child(NetworksContainer.class).child(Networks.class),
1159                     org.opendaylight.infrautils.utils.concurrent.Executors
1160                             .newListeningSingleThreadExecutor("NetworksReactor", LOG));
1161         }
1162
1163         public void add(InstanceIdentifier<Networks> iid, Networks val) {
1164             if (!isBGPEntityOwner()) {
1165                 return;
1166             }
1167             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1168             synchronized (BgpConfigurationManager.this) {
1169                 String rd = val.getRd();
1170                 String pfxlen = val.getPrefixLen();
1171                 String nh = val.getNexthop().getValue();
1172                 BgpRouter br = getClient(YANG_OBJ);
1173                 if (br == null) {
1174                     LOG.debug("{} Unable to process add for rd {} prefix {} nexthop {}; {} {}", YANG_OBJ, rd, pfxlen,
1175                             nh, BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1176                     return;
1177                 }
1178                 Long label = val.getLabel().toJava();
1179                 int lbl = label == null ? qbgpConstants.LBL_NO_LABEL
1180                         : label.intValue();
1181                 int l3vni = val.getL3vni() == null ? qbgpConstants.LBL_NO_LABEL
1182                         : val.getL3vni().intValue();
1183                 int l2vni = val.getL2vni() == null ? qbgpConstants.LBL_NO_LABEL
1184                         : val.getL2vni().intValue();
1185
1186                 BgpControlPlaneType protocolType = val.getBgpControlPlaneType();
1187                 int ethernetTag = val.getEthtag().intValue();
1188                 String esi = val.getEsi();
1189                 String macaddress = val.getMacaddress();
1190                 EncapType encapType = val.getEncapType();
1191                 String routerMac = val.getRoutermac();
1192
1193                 try {
1194                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni, BgpUtil.convertToThriftProtocolType(protocolType),
1195                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1196                 } catch (TException | BgpRouterException e) {
1197                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1198                 }
1199             }
1200         }
1201
1202         @Override
1203         public void remove(InstanceIdentifier<Networks> iid, Networks val) {
1204             if (!isBGPEntityOwner()) {
1205                 return;
1206             }
1207             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1208             synchronized (BgpConfigurationManager.this) {
1209                 String rd = val.getRd();
1210                 String pfxlen = val.getPrefixLen();
1211                 BgpRouter br = getClient(YANG_OBJ);
1212                 if (br == null) {
1213                     LOG.debug("{} Unable to process remove for rd {} prefix {}; {} {}", YANG_OBJ, rd, pfxlen,
1214                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1215                     return;
1216                 }
1217                 Long label = val.getLabel().toJava();
1218                 int lbl = label == null ? 0 : label.intValue();
1219                 if (rd == null && lbl > 0) {
1220                     //LU prefix is being deleted.
1221                     rd = Integer.toString(lbl);
1222                 }
1223                 try {
1224                     br.delPrefix(rd, pfxlen);
1225                 } catch (TException | BgpRouterException e) {
1226                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1227                 }
1228             }
1229         }
1230
1231         /**get the value AFI from a prefix as "x.x.x.x/x".
1232          *
1233          * @param pfxlen the prefix to get an afi
1234          * @return the afi value as you are need
1235          */
1236         public  int  testValueAFI(String pfxlen) {
1237             int afiNew = af_afi.AFI_IP.getValue();
1238             try {
1239                 String ipOnly = pfxlen.substring(0, pfxlen.lastIndexOf("/"));
1240                 java.net.Inet6Address.getByName(ipOnly);
1241                 afiNew = af_afi.AFI_IPV6.getValue();
1242             } catch (java.net.UnknownHostException e) {
1243                 //ce n'est pas de l'ipv6
1244             }
1245             return afiNew;
1246         }
1247
1248
1249         @Override
1250         public void update(final InstanceIdentifier<Networks> iid,
1251                 final Networks oldval, final Networks newval) {
1252             if (!isBGPEntityOwner()) {
1253                 return;
1254             }
1255             if (oldval.equals(newval)) {
1256                 //Update: OLD and New values are same, no need to trigger remove/add.
1257                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1258                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1259                 return;
1260             }
1261             LOG.debug("update networks old val RD: {}, Prefix: {}, Label: {}, NH: {} "
1262                             + "new val RD: {}, Prefix: {}, Label: {}, NH: {}",
1263                     oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop(),
1264                     newval.getRd(), newval.getPrefixLen(), newval.getLabel(), newval.getNexthop());
1265             remove(iid, oldval);
1266             add(iid, newval);
1267         }
1268     }
1269
1270     static Timer timer = new Timer();
1271
1272     public class VrfsReactor extends AbstractAsyncDataTreeChangeListener<Vrfs>
1273             implements ClusteredDataTreeChangeListener<Vrfs> {
1274
1275         private static final String YANG_OBJ = "vrfs ";
1276
1277         public VrfsReactor() {
1278             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
1279                     InstanceIdentifier.create(Bgp.class).child(VrfsContainer.class).child(Vrfs.class),
1280                     org.opendaylight.infrautils.utils.concurrent.Executors
1281                             .newListeningSingleThreadExecutor("VrfsReactor", LOG));
1282         }
1283
1284         @Override
1285         public void add(InstanceIdentifier<Vrfs> iid, Vrfs vrfs) {
1286             if (!isBGPEntityOwner()) {
1287                 return;
1288             }
1289             LOG.debug("received add Vrfs config value {}", vrfs.getRd());
1290             synchronized (BgpConfigurationManager.this) {
1291                 String rd = vrfs.getRd();
1292                 BgpRouter br = getClient(YANG_OBJ);
1293                 if (br == null) {
1294                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1295                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1296                     return;
1297                 }
1298                 try {
1299                     List<AddressFamiliesVrf> vrfAddrFamilyList = vrfs.getAddressFamiliesVrf();
1300                     for (AddressFamiliesVrf vrfAddrFamily : vrfAddrFamilyList) {
1301                         /*add to br the new vrfs arguments*/
1302                         br.addVrf(BgpUtil.getLayerType(vrfAddrFamily), rd, vrfs.getImportRts(),
1303                                 vrfs.getExportRts(), vrfAddrFamily.getAfi().toJava(), vrfAddrFamily.getSafi().toJava());
1304                     }
1305                     /*add to br the vrfs contained in mapNewAdFamily*/
1306                     List<AddressFamiliesVrf> vrfAddrFamilyListFromMap = mapNewAdFamily.get(rd);
1307                     if (vrfAddrFamilyListFromMap == null) {
1308                         return;
1309                     }
1310
1311                     for (AddressFamiliesVrf adf : vrfAddrFamilyListFromMap) {
1312                         if (vrfAddrFamilyList.contains(adf)) {
1313                             mapNewAdFamily.remove(rd);
1314                         } else  if (adf != null) {
1315
1316                             br.addVrf(BgpUtil.getLayerType(adf), rd, vrfs.getImportRts(),
1317                                     vrfs.getExportRts(), adf.getAfi().toJava(), adf.getSafi().toJava());
1318                             // remove AddressFamiliesVrf which was already added to BGP
1319                             vrfAddrFamilyListFromMap.remove(adf);
1320                             if (vrfAddrFamilyListFromMap.isEmpty()) {
1321                                 // remove Vrf entry from temp mapNewAdFamily if all its AddressFamiliesVrf was
1322                                 // added to BGP
1323                                 mapNewAdFamily.remove(rd);
1324                             }
1325                         }
1326                     }
1327                 } catch (TException | BgpRouterException e) {
1328                     LOG.error("{} get {}, Add received exception", YANG_OBJ, ADD_WARN, e);
1329                 }
1330             }
1331         }
1332
1333         @Override
1334         public void remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1335             if (!isBGPEntityOwner()) {
1336                 return;
1337             }
1338             LOG.debug("received remove Vrfs config val {}", val.getRd());
1339             synchronized (BgpConfigurationManager.this) {
1340                 String rd = val.getRd();
1341                 BgpRouter br = getClient(YANG_OBJ);
1342                 if (br == null) {
1343                     LOG.debug("{} Unable to process remove for rd {}; {} {}", YANG_OBJ, rd,
1344                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1345                     return;
1346                 }
1347                 try {
1348                     List<AddressFamiliesVrf> adf = mapNewAdFamily.get(rd);
1349                     adf = adf != null ? adf : new ArrayList<>();
1350                     for (AddressFamiliesVrf s : val.getAddressFamiliesVrf()) {
1351                         br.delVrf(rd, s.getAfi().toJava(), s.getSafi().toJava());
1352                         adf.remove(s);// remove in the map the vrf in waiting for advertise quagga
1353                     }
1354                     if (adf.isEmpty()) {
1355                         mapNewAdFamily.remove(rd);
1356                     }
1357                 } catch (TException | BgpRouterException e) {
1358                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1359                 }
1360             }
1361         }
1362
1363         @Override
1364         public void update(InstanceIdentifier<Vrfs> iid,
1365                 Vrfs oldval, Vrfs newval) {
1366             if (oldval != null && newval != null) {
1367                 LOG.debug("received update Vrfs config val {}, VRFS: Update getting triggered for VRFS rd {}",
1368                         newval.getRd(), oldval.getRd());
1369             } else {
1370                 LOG.debug("received update Vrfs config val {}, from old vrf {}",
1371                         newval, oldval);
1372             }
1373             if (!isBGPEntityOwner()) {
1374                 return;
1375             }
1376
1377             List<AddressFamiliesVrf> adFamilyVrfToDel = new ArrayList<>();
1378             List<AddressFamiliesVrf> adFamilyVrfToAdd = new ArrayList<>();
1379             List<AddressFamiliesVrf> oldlistAdFamilies = new ArrayList<>();
1380             List<AddressFamiliesVrf> newlistAdFamilies = new ArrayList<>();
1381             if (oldval != null) {
1382                 oldlistAdFamilies = oldval.getAddressFamiliesVrf() == null
1383                         ? new ArrayList<>() : oldval.getAddressFamiliesVrf();
1384             }
1385             if (newval != null) {
1386                 newlistAdFamilies = newval.getAddressFamiliesVrf() == null
1387                         ? new ArrayList<>() : newval.getAddressFamiliesVrf();
1388             }
1389             /*find old AddressFamily to remove from new configuration*/
1390             for (AddressFamiliesVrf adVrf : oldlistAdFamilies) {
1391                 if (!newlistAdFamilies.contains(adVrf)) {
1392                     adFamilyVrfToDel.add(adVrf);
1393                 }
1394             }
1395             /*find new AddressFamily to add to unexisting configuration*/
1396             for (AddressFamiliesVrf adVrf : newlistAdFamilies) {
1397                 if (!oldlistAdFamilies.contains(adVrf)) {
1398                     adFamilyVrfToAdd.add(adVrf);
1399                 }
1400             }
1401             String rd = newval != null ? newval.getRd() : null;
1402             if (rd != null) {
1403                 BgpRouter br = getClient(YANG_OBJ);
1404                 if (br == null) {
1405                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1406                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1407                     return;
1408                 }
1409
1410                 for (AddressFamiliesVrf adfvrf : adFamilyVrfToAdd) {
1411                     try {
1412                         LOG.debug("call addVRf rd {} afi {} safi {}", rd, adfvrf.getAfi(), adfvrf.getSafi());
1413                         br.addVrf(BgpUtil.getLayerType(adfvrf), rd, newval.getImportRts(),
1414                                 newval.getExportRts(),adfvrf.getAfi().toJava(), adfvrf.getSafi().toJava());
1415                     } catch (TException | BgpRouterException e) {
1416                         LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1417                     }
1418                 }
1419
1420                 for (AddressFamiliesVrf adfToDel : adFamilyVrfToDel) {
1421                     try {
1422                         LOG.debug("call delVRf rd {} afi {} safi {}", rd, adfToDel.getAfi(), adfToDel.getSafi());
1423                         br.delVrf(rd, adfToDel.getAfi().toJava(), adfToDel.getSafi().toJava());
1424                     } catch (TException | BgpRouterException e) {
1425                         LOG.error("{} delVrf received exception; {}", YANG_OBJ, ADD_WARN, e);
1426                     }
1427                 }
1428             }
1429         }
1430     }
1431
1432     public class BgpReactor extends AbstractAsyncDataTreeChangeListener<Bgp>
1433             implements ClusteredDataTreeChangeListener<Bgp> {
1434
1435         private static final String YANG_OBJ = "Bgp ";
1436
1437         public BgpReactor() {
1438             super(dataBroker, LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(Bgp.class),
1439                     org.opendaylight.infrautils.utils.concurrent.Executors
1440                             .newListeningSingleThreadExecutor("BgpReactor", LOG));
1441         }
1442
1443         @Override
1444         public void add(InstanceIdentifier<Bgp> iid, Bgp val) {
1445             LOG.debug("received add Bgp config");
1446
1447             try {
1448                 initer.await();
1449             } catch (InterruptedException e) {
1450                 // Ignored
1451             }
1452             synchronized (BgpConfigurationManager.this) {
1453                 config = val;
1454                 if (!isBGPEntityOwner()) {
1455                     return;
1456                 }
1457             }
1458         }
1459
1460         @Override
1461         public void remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1462             if (!isBGPEntityOwner()) {
1463                 return;
1464             }
1465             LOG.debug("received remove Bgp config");
1466
1467             config = null;
1468         }
1469
1470         @Override
1471         public void update(InstanceIdentifier<Bgp> iid,
1472                 Bgp oldval, Bgp newval) {
1473             if (!isBGPEntityOwner()) {
1474                 return;
1475             }
1476
1477             config = newval;
1478         }
1479     }
1480
1481     @SuppressWarnings("deprecation")
1482     public class MultipathReactor extends AbstractAsyncDataTreeChangeListener<Multipath>
1483             implements ClusteredDataTreeChangeListener<Multipath> {
1484
1485         private static final String YANG_OBJ = "multipath ";
1486
1487         public MultipathReactor() {
1488             super(dataBroker, LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(Bgp.class)
1489                     .child(MultipathContainer.class).child(Multipath.class),
1490                     org.opendaylight.infrautils.utils.concurrent.Executors
1491                             .newListeningSingleThreadExecutor("MultipathReactor", LOG));
1492         }
1493
1494         @Override
1495         public void remove(InstanceIdentifier<Multipath> iid, Multipath val) {
1496             executor.execute(new MultipathStatusChange(val));
1497         }
1498
1499         @Override
1500         public void update(InstanceIdentifier<Multipath> iid, Multipath oldval, Multipath newval) {
1501             executor.execute(new MultipathStatusChange(newval));
1502         }
1503
1504         @Override
1505         public void add(InstanceIdentifier<Multipath> key, Multipath dataObjectModification) {
1506             executor.execute(new MultipathStatusChange(dataObjectModification));
1507         }
1508
1509         class MultipathStatusChange implements Runnable {
1510
1511             Multipath multipath;
1512
1513             MultipathStatusChange(Multipath multipath) {
1514                 this.multipath = multipath;
1515             }
1516
1517             @Override
1518             public void run() {
1519                 if (isBGPEntityOwner()) {
1520                     synchronized (BgpConfigurationManager.this) {
1521
1522                         BgpRouter br = getClient(YANG_OBJ);
1523
1524                         if (br != null) {
1525                             af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1526                             af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1527
1528                             try {
1529                                 if (multipath.isMultipathEnabled()) {
1530                                     br.enableMultipath(afi, safi);
1531                                 } else {
1532                                     br.disableMultipath(afi, safi);
1533                                 }
1534                             } catch (TException | BgpRouterException e) {
1535                                 LOG.error("{} received exception", YANG_OBJ, e);
1536                             }
1537                         }
1538                     }
1539                 }
1540             }
1541
1542         }
1543
1544         @Override
1545         public void close() {
1546             super.close();
1547         }
1548     }
1549
1550     @SuppressWarnings("deprecation")
1551     public class VrfMaxpathReactor extends AbstractAsyncDataTreeChangeListener<VrfMaxpath>
1552             implements ClusteredDataTreeChangeListener<VrfMaxpath> {
1553
1554         private static final String YANG_OBJ = "vrfMaxpath ";
1555
1556         public VrfMaxpathReactor() {
1557             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
1558                     InstanceIdentifier.create(Bgp.class).child(VrfMaxpathContainer.class).child(VrfMaxpath.class),
1559                     org.opendaylight.infrautils.utils.concurrent.Executors
1560                             .newListeningSingleThreadExecutor("VrfMaxpathReactor", LOG));
1561         }
1562
1563         class VrfMaxPathConfigurator implements Runnable {
1564
1565             VrfMaxpath vrfMaxpathVal;
1566
1567             VrfMaxPathConfigurator(VrfMaxpath vrfMaxPathVal) {
1568                 this.vrfMaxpathVal = vrfMaxPathVal;
1569             }
1570
1571             @Override
1572             public void run() {
1573                 if (isBGPEntityOwner()) {
1574                     synchronized (BgpConfigurationManager.this) {
1575                         BgpRouter br = getClient(YANG_OBJ);
1576                         if (br != null) {
1577                             try {
1578                                 br.multipaths(vrfMaxpathVal.getRd(), vrfMaxpathVal.getMaxpaths().toJava());
1579                                 LOG.debug("Maxpath for vrf {} is {}", vrfMaxpathVal.getRd(),
1580                                         vrfMaxpathVal.getMaxpaths());
1581                             } catch (TException | BgpRouterException e) {
1582                                 LOG.error("{} received exception", YANG_OBJ, e);
1583                             }
1584                         }
1585                     }
1586                 }
1587             }
1588         }
1589
1590         @Override
1591         public void remove(InstanceIdentifier<VrfMaxpath> iid, VrfMaxpath vrfMaxPathVal) {
1592             if (isBGPEntityOwner()) {
1593                 synchronized (BgpConfigurationManager.this) {
1594                     BgpRouter br = getClient(YANG_OBJ);
1595                     if (br != null) {
1596                         try {
1597                             br.multipaths(vrfMaxPathVal.getRd(), BgpConstants.BGP_DEFAULT_MULTIPATH);
1598                             LOG.debug("Del Maxpath for vrf: {} ", vrfMaxPathVal.getRd());
1599                         } catch (TException | BgpRouterException e) {
1600                             LOG.error(YANG_OBJ + " del received exception:", e);
1601                         }
1602                     }
1603                 }
1604             }
1605         }
1606
1607         @Override
1608         public void update(InstanceIdentifier<VrfMaxpath> iid,
1609                               VrfMaxpath oldval, VrfMaxpath newval) {
1610             if (!Objects.equals(oldval.getMaxpaths(), newval.getMaxpaths())) {
1611                 executor.execute(new VrfMaxPathConfigurator(newval));
1612             }
1613         }
1614
1615         @Override
1616         public void add(InstanceIdentifier<VrfMaxpath> instanceIdentifier, VrfMaxpath vrfMaxpathVal) {
1617             executor.execute(new VrfMaxPathConfigurator(vrfMaxpathVal));
1618         }
1619
1620         @Override
1621         public void close() {
1622             super.close();
1623         }
1624     }
1625
1626     public class BfdConfigReactor extends AbstractAsyncDataTreeChangeListener<BfdConfig>
1627             implements ClusteredDataTreeChangeListener<BfdConfig> {
1628
1629         private static final String YANG_OBJ = "BfdConfig ";
1630
1631         public BfdConfigReactor() {
1632             super(dataBroker, LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(BfdConfig.class),
1633                     org.opendaylight.infrautils.utils.concurrent.Executors
1634                             .newListeningSingleThreadExecutor("BfdConfigReactor", LOG));
1635         }
1636
1637         @Override
1638         public void add(InstanceIdentifier<BfdConfig> iid, BfdConfig val) {
1639             if (!isBGPEntityOwner()) {
1640                 return;
1641             }
1642             BgpRouter br = getClient(YANG_OBJ);
1643             LOG.debug("received bfd config: bfd enabled {} min-rx {} min-tx {} detect-mul {} mhop {}",
1644                     val.isBfdEnabled(), val.getMinRx(), val.getMinTx(),
1645                     val.getDetectMult(), val.isMultihop());
1646             if (br == null) {
1647                 LOG.debug(YANG_OBJ + "{} Unable to process add  {}",
1648                         BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1649                 return;
1650             }
1651             if (val.isBfdEnabled() == false) {
1652                 LOG.debug("BFD not enabled. Ignoring the config add");
1653                 return;
1654             }
1655             int minRx = val.getMinRx().intValue();
1656             int minTx = val.getMinTx().intValue();
1657             int detectMult = val.getDetectMult().intValue();
1658             boolean multiHop = val.isMultihop();
1659             try {
1660                 br.addBfd(detectMult, minRx, minTx,multiHop);
1661             } catch (TException | BgpRouterException e) {
1662                 LOG.error("{} get {}, Add received exception;", YANG_OBJ, ADD_WARN, e);
1663             }
1664         }
1665
1666         @Override
1667         public void remove(InstanceIdentifier<BfdConfig> iid, BfdConfig val) {
1668             if (!isBGPEntityOwner()) {
1669                 return;
1670             }
1671             LOG.debug("received bfd config removal");
1672             BgpRouter br = getClient(YANG_OBJ);
1673             if (br == null) {
1674                 LOG.debug("{} Unable to process del {}  {}", YANG_OBJ,
1675                         BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1676                 return;
1677             }
1678             try {
1679                 br.delBfd();
1680             } catch (TException | BgpRouterException e) {
1681                 LOG.error("{} get {}, Del received exception;", YANG_OBJ, ADD_WARN, e);
1682             }
1683
1684         }
1685
1686         @Override
1687         public void update(InstanceIdentifier<BfdConfig> iid,
1688                               BfdConfig oldval, BfdConfig newval) {
1689             LOG.debug("received bfd config: updated oldval bfd enabled {}"
1690                     + "min-rx {} min-tx {} detect-mul {} mhop {}",
1691                     oldval.isBfdEnabled(), oldval.getMinRx(), oldval.getMinTx(),
1692                     oldval.getDetectMult(), oldval.isMultihop());
1693             LOG.debug("received bfd config: updated newval bfd enabled {}"
1694                     + "min-rx {} min-tx {} detect-mul {} mhop {}",
1695                     newval.isBfdEnabled(), newval.getMinRx(), newval.getMinTx(),
1696                     newval.getDetectMult(), newval.isMultihop());
1697             if (oldval.isBfdEnabled()) {
1698                 LOG.debug("deleting bfd config on an update");
1699                 remove(iid, oldval);
1700             }
1701             LOG.debug("adding bfd config on an update");
1702             add(iid, newval);
1703         }
1704     }
1705
1706
1707     public boolean isIpAvailable(String odlip) {
1708
1709         try {
1710             if (odlip != null) {
1711                 if ("127.0.0.1".equals(odlip)) {
1712                     return true;
1713                 }
1714                 Enumeration<NetworkInterface> networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces();
1715                 while (networkInterfaceEnumeration.hasMoreElements()) {
1716                     NetworkInterface networkInterface = networkInterfaceEnumeration.nextElement();
1717                     Enumeration<InetAddress>  inetAddressEnumeration = networkInterface.getInetAddresses();
1718                     while (inetAddressEnumeration.hasMoreElements()) {
1719                         InetAddress inetAddress = inetAddressEnumeration.nextElement();
1720                         if (odlip.equals(inetAddress.getHostAddress())) {
1721                             return true;
1722                         }
1723                     }
1724                 }
1725             }
1726         } catch (SocketException e) {
1727             // Ignored?
1728         }
1729         return false;
1730     }
1731
1732     public long getStalePathtime(int defValue, AsId asId) {
1733         long spt = 0;
1734         try {
1735             InstanceIdentifier<GracefulRestart> id =
1736                     InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
1737             Optional<GracefulRestart> gracefulRestartOptional = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1738                     LogicalDatastoreType.CONFIGURATION, id);
1739             if (gracefulRestartOptional.isPresent()) {
1740                 spt = gracefulRestartOptional.get().getStalepathTime().toJava();
1741             }
1742         } catch (NullPointerException e) {
1743             try {
1744                 spt = asId.getStalepathTime().toJava();
1745                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1746             } catch (NullPointerException ignore) {
1747                 LOG.trace("BGP AS id is not set using graceful");
1748                 spt = defValue;
1749             }
1750         } catch (InterruptedException | ExecutionException e) {
1751             LOG.trace("Exception while reading GracefulRestart DS for the As {}", asId.getStalepathTime());
1752             spt = defValue;
1753         }
1754         if (spt == 0) {
1755             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1756             spt = defValue;
1757         }
1758         return spt;
1759     }
1760
1761     public static boolean isValidConfigBgpHostPort(String bgpHost, int bgpPort) {
1762         if (!bgpHost.equals(DEF_CHOST)) {
1763             return true;
1764         } else {
1765             return false;
1766         }
1767     }
1768
1769     public synchronized void bgpRestarted() {
1770         /*
1771          * If there a thread which in the process of stale cleanup, cancel it
1772          * and start a new thread (to avoid processing same again).
1773          */
1774         if (previousReplayJobInProgress()) {
1775             cancelPreviousReplayJob();
1776         }
1777         Runnable task = () -> {
1778             try {
1779                 LOG.info("running bgp replay task ");
1780                 if (get() == null) {
1781                     String host = getConfigHost();
1782                     int port = getConfigPort();
1783                     LOG.info("connecting  to bgp host {} ", host);
1784                     bgpRouter.connect(host, port);
1785                     LOG.info("no config to push in bgp replay task ");
1786                     return;
1787                 }
1788                 setStaleStartTime(System.currentTimeMillis());
1789                 LOG.info("started creating stale fibDSWriter  map ");
1790                 createStaleFibMap();
1791                 setStaleEndTime(System.currentTimeMillis());
1792                 LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
1793                 LOG.info("started bgp config replay ");
1794                 setCfgReplayStartTime(System.currentTimeMillis());
1795                 boolean replaySucceded = replay();
1796                 setCfgReplayEndTime(System.currentTimeMillis());
1797                 LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1798                 if (replaySucceded) {
1799                     long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1800                     setStaleCleanupTime(routeSyncTime);
1801                     LOG.error("starting the stale cleanup timer: {} seconds", routeSyncTime);
1802                     routeCleanupFuture = executor.schedule(new RouteCleanup(), routeSyncTime, TimeUnit.SECONDS);
1803                 } else {
1804                     LOG.error("skipping stale cleanup, may be due to exception while replay");
1805                     staledFibEntriesMap.clear();
1806                 }
1807             } catch (InterruptedException | TimeoutException | ExecutionException eCancel) {
1808                 LOG.error("Stale Cleanup Task Cancelled", eCancel);
1809             }
1810         };
1811         lastReplayJobFt = executor.submit(task);
1812     }
1813
1814     private boolean previousReplayJobInProgress() {
1815         return ((lastReplayJobFt != null && !lastReplayJobFt.isDone())
1816                 || (routeCleanupFuture != null && !routeCleanupFuture.isDone()));
1817     }
1818
1819     private void cancelPreviousReplayJob() {
1820         try {
1821             LOG.error("cancelling already running bgp replay task");
1822             if (lastReplayJobFt != null) {
1823                 lastReplayJobFt.cancel(true);
1824                 lastReplayJobFt = null;
1825                 staledFibEntriesMap.clear();
1826             }
1827             if (routeCleanupFuture != null) {
1828                 routeCleanupFuture.cancel(true);
1829                 routeCleanupFuture = null;
1830                 staledFibEntriesMap.clear();
1831             }
1832             Thread.sleep(2000);
1833         } catch (InterruptedException e) {
1834             LOG.error("Failed to cancel previous replay job ", e);
1835         }
1836     }
1837
1838     private void doRouteSync() {
1839         for (af_afi afi : af_afi.values()) {
1840             try {
1841                 bgpRouter.initRibSync(bgpSyncHandle);
1842             } catch (BgpRouterException e) {
1843                 LOG.error("Route sync aborted, exception when initializing", e);
1844                 return;
1845             }
1846             LOG.error("Starting BGP route sync for afi {}", afi.getValue());
1847             while (bgpSyncHandle.getState() != BgpSyncHandle.DONE) {
1848                 Routes routes = null;
1849                 int noUpdates = 0;
1850                 try {
1851                     routes = bgpRouter.doRibSync(bgpSyncHandle, afi);
1852                 } catch (TException | BgpRouterException e) {
1853                     LOG.error("Route sync aborted, exception when syncing", e);
1854                     return;
1855                 }
1856                 Iterator<Update> updates = routes.getUpdatesIterator();
1857                 while (updates.hasNext()) {
1858                     noUpdates++;
1859                     Update update = updates.next();
1860                     String rd = update.getRd();
1861                     String nexthop = update.getNexthop();
1862
1863                     // TODO: decide correct label here
1864                     int label = update.getL3label();
1865                     int l2label = update.getL2label();
1866
1867                     String prefix = update.getPrefix();
1868                     int plen = update.getPrefixlen();
1869
1870
1871                     // TODO: protocol type will not be available in "update"
1872                     // use "rd" to query vrf table and obtain the protocol_type.
1873                     // Currently using PROTOCOL_EVPN as default.
1874                     onUpdatePushRoute(
1875                            protocol_type.PROTOCOL_L3VPN,
1876                            rd,
1877                            prefix,
1878                            plen,
1879                            nexthop,
1880                            update.getMacaddress(),
1881                            Uint32.valueOf(label),
1882                            Uint32.valueOf(l2label),
1883                            update.getRoutermac(),
1884                            afi);
1885                 }
1886                 LOG.error("No of updates for afi {} is {}", afi.getValue(), noUpdates);
1887             }
1888         }
1889         try {
1890             LOG.error("Ending BGP route-sync");
1891             bgpRouter.endRibSync(bgpSyncHandle);
1892         } catch (BgpRouterException e) {
1893             LOG.error("Route sync aborted, exception when ending", e);
1894         }
1895     }
1896
1897     public void addTepToElanDS(String rd, String tepIp, String mac, Uint32 l2vni) {
1898         boolean needUpdate = addToRt2TepMap(rd, tepIp, mac, l2vni);
1899         if (needUpdate) {
1900             LOG.info("Adding tepIp {} with RD {} to ELan DS", tepIp, rd);
1901             bgpUtil.addTepToElanInstance(rd, tepIp);
1902         } else {
1903             LOG.debug("Skipping the Elan update for RT2 from tep {} rd {}", tepIp, rd);
1904         }
1905     }
1906
1907     public void deleteTepfromElanDS(String rd, String tepIp, String mac) {
1908         boolean needUpdate = deleteFromRt2TepMap(rd, tepIp, mac);
1909         if (needUpdate) {
1910             LOG.info("Deleting tepIp {} with RD {} to ELan DS", tepIp, rd);
1911             bgpUtil.deleteTepFromElanInstance(rd, tepIp);
1912         } else {
1913             LOG.debug("Skipping the Elan update for RT2 withdraw from tep {} rd {}", tepIp, rd);
1914         }
1915     }
1916
1917     /* onUpdatePushRoute
1918      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1919      *  - Entry compare shall include NextHop, Label.
1920      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1921      *  - If entry not found, add to FIB Config DS.
1922      *  - If entry found, but either Label/NextHop doesn't match.
1923      *      - Update FIB Config DS with modified values.
1924      *      - delete from Stale Map.
1925      */
1926
1927     public void onUpdatePushRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
1928                                   String macaddress, Uint32 label, Uint32 l2label, String routermac, af_afi afi) {
1929         PrefixUpdateEvent prefixUpdateEvent = new PrefixUpdateEvent(protocolType,rd,prefix,plen,nextHop,
1930                 macaddress,label,l2label,routermac,afi);
1931         bgpUpdatesHistory.addToHistory(TransactionType.ADD, prefixUpdateEvent);
1932         boolean addroute = false;
1933         boolean macupdate = false;
1934         Uint32 l3vni = Uint32.ZERO;
1935         VrfEntry.EncapType encapType = VrfEntry.EncapType.Mplsgre;
1936         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1937             encapType = VrfEntry.EncapType.Vxlan;
1938             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
1939             if (vpnInstanceOpDataEntry != null) {
1940                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
1941                     LOG.info("Got RT2 route for RD {} l3label {} l2label {} from tep {} with mac {} remote RD {}",
1942                             vpnInstanceOpDataEntry.getVpnInstanceName(), label, l2label, nextHop, macaddress, rd);
1943                     addTepToElanDS(rd, nextHop, macaddress, l2label);
1944                     macupdate = true;
1945                 } else {
1946                     l3vni = vpnInstanceOpDataEntry.getL3vni();
1947                 }
1948             } else {
1949                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1950                 return;
1951             }
1952         }
1953
1954         if (!staledFibEntriesMap.isEmpty()) {
1955             // restart Scenario, as MAP is not empty.
1956             Map<String, Uint32> map = staledFibEntriesMap.get(rd);
1957             if (map != null) {
1958                 String prefixNextHop = appendNextHopToPrefix(prefix + "/" + plen, nextHop);
1959                 Uint32 labelInStaleMap = map.get(prefixNextHop);
1960                 if (null == labelInStaleMap) {
1961                     // New Entry, which happened to be added during restart.
1962                     addroute = true;
1963                 } else {
1964                     map.remove(prefixNextHop);
1965                     if (isRouteModified(label, labelInStaleMap)) {
1966                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1967                         // Existing entry, where in Label got modified during restart
1968                         addroute = true;
1969                     }
1970                 }
1971             } else {
1972                 LOG.debug("rd {} map is null while processing prefix {} ", rd, prefix);
1973                 addroute = true;
1974             }
1975         } else {
1976             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1977             addroute = true;
1978         }
1979         if (macupdate) {
1980             LOG.info("ADD: Adding Mac Fib entry rd {} mac{} nexthop {} l2vni {}", rd, macaddress, nextHop, l2label);
1981             fibDSWriter.addMacEntryToDS(rd, macaddress, prefix, Collections.singletonList(nextHop),
1982                     encapType, l2label, routermac, RouteOrigin.BGP);
1983             LOG.info("ADD: Added Mac Fib entry rd {} prefix {} nexthop {} label {}", rd, macaddress, nextHop, l2label);
1984         } else if (addroute) {
1985             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {} afi {}",
1986                     rd, prefix, nextHop, label, afi);
1987             // TODO: modify addFibEntryToDS signature
1988             List<String> nextHopList = Collections.singletonList(nextHop);
1989             fibDSWriter.addFibEntryToDS(rd, prefix + "/" + plen, nextHopList, encapType, label, l3vni,
1990                                         routermac, RouteOrigin.BGP);
1991             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1992             String vpnName = bgpUtil.getVpnNameFromRd(rd);
1993             if (vpnName != null) {
1994                 vpnLinkService.leakRouteIfNeeded(vpnName, prefix, nextHopList, label, RouteOrigin.BGP,
1995                                                  NwConstants.ADD_FLOW);
1996             }
1997         }
1998     }
1999
2000     public void onUpdateWithdrawRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
2001             String macaddress) {
2002         PrefixWithdrawEvent prefixWithdrawEvent = new PrefixWithdrawEvent(protocolType,rd,prefix,plen,
2003                 nextHop,macaddress);
2004         bgpUpdatesHistory.addToHistory(TransactionType.ADD, prefixWithdrawEvent);
2005         boolean macupdate = false;
2006         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
2007             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
2008             if (vpnInstanceOpDataEntry != null) {
2009                 Uint32 vni = vpnInstanceOpDataEntry.getL3vni();
2010                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
2011                     LOG.debug("Got RT2 withdraw for RD {} {} from tep {} with mac {} remote RD {}",
2012                             vpnInstanceOpDataEntry.getVpnInstanceName(), vni, nextHop, macaddress, rd);
2013                     deleteTepfromElanDS(rd, nextHop, macaddress);
2014                     LOG.debug("For rd {}. skipping fib update", rd);
2015                     macupdate = true;
2016                 }
2017             } else {
2018                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
2019                 return;
2020             }
2021         }
2022         if (macupdate) {
2023             LOG.info("Removing Mac Fib entry rd {} mac{} nexthop {} ", rd, macaddress, nextHop);
2024             fibDSWriter.removeMacEntryFromDS(rd, macaddress);
2025             LOG.info("Removed Mac Fib entry rd {} prefix {} nexthop {} ", rd, macaddress, nextHop);
2026         } else {
2027             LOG.info("REMOVE: Removing Fib entry rd {} prefix {}", rd, prefix);
2028             fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nextHop);
2029             LOG.info("REMOVE: Removed Fib entry rd {} prefix {}", rd, prefix);
2030         }
2031     }
2032
2033     //TODO: below function is for testing purpose with cli
2034     public void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
2035         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
2036         fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nexthop);
2037         String vpnName = bgpUtil.getVpnNameFromRd(rd);
2038         if (vpnName != null) {
2039             vpnLinkService.leakRouteIfNeeded(vpnName, prefix, null /*nextHopList*/, Uint32.ZERO /*INVALID_LABEL*/,
2040                                              RouteOrigin.BGP, NwConstants.DEL_FLOW);
2041         }
2042     }
2043
2044     public void peerDown(String ipAddress, long asNumber) {
2045         PeerDownEvent peerDownEvent = new PeerDownEvent(ipAddress,asNumber);
2046         bgpUpdatesHistory.addToHistory(TransactionType.ADD, peerDownEvent);
2047         List<String> tepIpList = bgpUtil.getDcgwTepConfig(ipAddress);
2048         if (tepIpList == null) {
2049             LOG.error("No Tep IP configured for DCGW {} on a peerDown", ipAddress);
2050             return;
2051         }
2052         tepIpList.forEach(tepIp -> {
2053             bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW);
2054         });
2055     }
2056
2057     public void peerUp(String ipAddress, long asNumber) {
2058         PeerUpEvent peerUpEvent = new PeerUpEvent(ipAddress,asNumber);
2059         bgpUpdatesHistory.addToHistory(TransactionType.ADD, peerUpEvent);
2060         List<String> tepIpList = bgpUtil.getDcgwTepConfig(ipAddress);
2061         if (tepIpList == null) {
2062             LOG.error("No Tep IP configured for DCGW {} on a peerUp", ipAddress);
2063             return;
2064         }
2065         tepIpList.forEach(tepIp -> {
2066             bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW);
2067         });
2068     }
2069
2070     private static boolean isRouteModified(Uint32 label, Uint32 labelInStaleMap) {
2071         return labelInStaleMap != null && !labelInStaleMap.equals(label);
2072     }
2073
2074     static class ReplayNbr {
2075         Neighbors nbr;
2076         boolean shouldRetry = false;
2077
2078         public Neighbors getNbr() {
2079             return nbr;
2080         }
2081
2082         public boolean isShouldRetry() {
2083             return shouldRetry;
2084         }
2085
2086         public void setShouldRetry(boolean retryNbr) {
2087             this.shouldRetry = retryNbr;
2088         }
2089
2090         ReplayNbr(Neighbors nbr, boolean shouldRetry) {
2091             this.nbr = nbr;
2092             this.shouldRetry = shouldRetry;
2093         }
2094     }
2095
2096     private static boolean replayNbrConfig(List<Neighbors> neighbors, BgpRouter br) {
2097         if (neighbors == null || neighbors.isEmpty()) {
2098             LOG.error("Replaying nbr configuration, received NULL list ");
2099             return true;
2100         }
2101
2102         List<ReplayNbr> replayNbrList = new ArrayList<>();
2103         for (Neighbors nbr : neighbors) {
2104             if (nbr != null) {
2105                 replayNbrList.add(new ReplayNbr(nbr, true));
2106             }
2107         }
2108         final int numberOfNbrRetries = 3;
2109         RetryOnException nbrRetry = new RetryOnException(numberOfNbrRetries);
2110         do {
2111             for (ReplayNbr replayNbr : replayNbrList) {
2112                 if (!replayNbr.isShouldRetry()) {
2113                     continue;
2114                 }
2115                 boolean replayDone = false;
2116                 LOG.debug("Replaying addNbr {}", replayNbr.getNbr().getAddress().getValue());
2117                 replayDone = false;
2118                 try {
2119                     final String md5password = extractMd5Secret(replayNbr.getNbr());
2120                     br.addNeighbor(replayNbr.getNbr().getAddress().getValue(),
2121                             replayNbr.getNbr().getRemoteAs().longValue(), md5password);
2122                     replayDone = true;
2123                 } catch (TApplicationException tae) {
2124                     LOG.debug("Replaying addNbr {}, tapplicationexception: ",
2125                             replayNbr.getNbr().getAddress().getValue(), tae);
2126                     if (tae.getType() == BgpRouterException.BGP_ERR_PEER_EXISTS) {
2127                         LOG.debug("Replaying addNbr Neighbor already present");
2128                         replayDone = true;
2129                     } else {
2130                         LOG.error("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), tae);
2131                     }
2132                 } catch (TException | BgpRouterException eNbr) {
2133                     LOG.debug("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), eNbr);
2134                 }
2135
2136                 LOG.debug("Replay addNbr {} successful", replayNbr.getNbr().getAddress().getValue());
2137
2138                 //Update Source handling
2139                 UpdateSource us = replayNbr.getNbr().getUpdateSource();
2140                 if (us != null) {
2141                     LOG.debug("Replaying updatesource {} to peer {}", us.getSourceIp().getValue(),
2142                             us.getPeerIp().getValue());
2143                     try {
2144                         br.addUpdateSource(us.getPeerIp().getValue(),
2145                                 us.getSourceIp().getValue());
2146                     } catch (TException | BgpRouterException eUs) {
2147                         LOG.debug("Replaying UpdateSource for Nbr {}, exception:",
2148                                 replayNbr.getNbr().getAddress().getValue(), eUs);
2149                     }
2150                     LOG.debug("Replay updatesource {} successful", us.getSourceIp().getValue());
2151                 }
2152                 //Ebgp Multihope
2153                 EbgpMultihop en = replayNbr.getNbr().getEbgpMultihop();
2154                 if (en != null) {
2155                     try {
2156                         br.addEbgpMultihop(en.getPeerIp().getValue(),
2157                                 en.getNhops().intValue());
2158                     } catch (TException | BgpRouterException eEbgpMhop) {
2159                         LOG.debug("Replaying EbgpMultihop for Nbr {}, exception: ",
2160                                 replayNbr.getNbr().getAddress().getValue(), eEbgpMhop);
2161                     }
2162                 }
2163
2164                 //afs
2165                 List<AddressFamilies> afs = replayNbr.getNbr().getAddressFamilies();
2166                 if (afs != null) {
2167                     for (AddressFamilies af : afs) {
2168                         af_afi afi = af_afi.findByValue(af.getAfi().intValue());
2169                         af_safi safi = af_safi.findByValue(af.getSafi().intValue());
2170                         try {
2171                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
2172                         } catch (TException | BgpRouterException eAFs) {
2173                             LOG.debug("Replaying AddressFamily for Nbr {}, exception:",
2174                                     replayNbr.getNbr().getAddress().getValue(), eAFs);
2175                         }
2176                     }
2177                 }
2178                 //replay is success --> no need to replay this nbr in next iteration.
2179                 replayNbr.setShouldRetry(replayDone ? false : true);
2180             }
2181         } while (nbrRetry.decrementAndRetry());
2182         boolean replaySuccess = true;
2183         for (ReplayNbr replayNbr : replayNbrList) {
2184             replaySuccess = replaySuccess && !replayNbr.isShouldRetry();
2185             if (replaySuccess == false) {
2186                 LOG.error("replayNbrConfig: will be cancelling stale cleanup, cfg nbr: {} Failed:",
2187                         replayNbr.getNbr().getAddress().getValue());
2188             }
2189         }
2190         return replaySuccess;
2191     }
2192
2193     public String getConfigHost() {
2194         if (config == null) {
2195             return hostStartup;
2196         }
2197         ConfigServer ts = config.getConfigServer();
2198         return ts == null ? hostStartup : ts.getHost().getValue();
2199     }
2200
2201     public int getConfigPort() {
2202         if (config == null) {
2203             return Integer.parseInt(portStartup);
2204         }
2205         ConfigServer ts = config.getConfigServer();
2206         return ts == null ? Integer.parseInt(portStartup) :
2207                 ts.getPort().intValue();
2208     }
2209
2210     public Bgp getConfig() {
2211         AtomicInteger bgpDSretryCount = new AtomicInteger(DS_RETRY_COUNT);
2212         while (0 != bgpDSretryCount.decrementAndGet()) {
2213             try {
2214                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
2215                         InstanceIdentifier.create(Bgp.class)).orElse(null);
2216             } catch (InterruptedException | ExecutionException e) {
2217                 //Config DS may not be up, so sleep for 1 second and retry
2218                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
2219                 try {
2220                     Thread.sleep(WAIT_TIME_BETWEEN_EACH_TRY_MILLIS);
2221                 } catch (InterruptedException timerEx) {
2222                     LOG.debug("WAIT_TIME_BETWEEN_EACH_TRY_MILLIS, Timer got interrupted while waiting for"
2223                             + "config DS availability", timerEx);
2224                 }
2225             }
2226         }
2227         LOG.error("failed to get bgp config");
2228         return null;
2229     }
2230
2231     @SuppressWarnings("checkstyle:IllegalCatch")
2232     public synchronized boolean replay() throws InterruptedException, TimeoutException, ExecutionException {
2233         boolean replaySucceded = true;
2234         boolean doRouteSync = false;
2235         String host = getConfigHost();
2236         int port = getConfigPort();
2237         LOG.error("connecting  to bgp host {} ", host);
2238         boolean res = bgpRouter.connect(host, port);
2239         if (!res) {
2240             LOG.error("Cannot connect to BGP config server at {} {}", host, port);
2241             return replaySucceded;
2242         }
2243         config = getConfig();
2244         if (config == null) {
2245             LOG.error("bgp config is empty nothing to push to bgp");
2246             return replaySucceded;
2247         }
2248         BgpRouter br = bgpRouter;
2249         AsId asId = config.getAsId();
2250         if (asId == null) {
2251             LOG.error("bgp as-id is null");
2252             return replaySucceded;
2253         }
2254         long asNum = asId.getLocalAs().toJava();
2255         IpAddress routerId = asId.getRouterId();
2256         String rid = routerId == null ? "" : routerId.stringValue();
2257         int stalepathTime = (int) getStalePathtime(bgpGrRestartTime, config.getAsId());
2258         boolean announceFbit = true;
2259         boolean replayDone = false;
2260         final int numberOfStartBgpRetries = 3;
2261         RetryOnException startBgpRetry = new RetryOnException(numberOfStartBgpRetries);
2262         do {
2263             try {
2264                 LOG.debug("Replaying BGPConfig ");
2265                 br.startBgp(asNum, rid, bgpKaTime, bgpHoldTime, stalepathTime, announceFbit);
2266                 LOG.debug("Replay BGPConfig successful");
2267                 replayDone = true;
2268                 break;
2269             } catch (BgpRouterException bre) {
2270                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
2271                     LOG.debug("Starting the routesync for exception", bre);
2272                     startBgpRetry.errorOccured();
2273                     if (!startBgpRetry.shouldRetry()) {
2274                         doRouteSync = true;
2275                         replayDone = true;
2276                     }
2277                 } else {
2278                     LOG.error("Replay: startBgp() received exception error {} : ",
2279                             bre.getErrorCode(), bre);
2280                     startBgpRetry.errorOccured();
2281                 }
2282             } catch (TApplicationException tae) {
2283                 if (tae.getType() == BgpRouterException.BGP_ERR_ACTIVE) {
2284                     LOG.debug("Starting the routesync for exception", tae);
2285                     startBgpRetry.errorOccured();
2286                     if (!startBgpRetry.shouldRetry()) {
2287                         doRouteSync = true;
2288                         replayDone = true;
2289                     }
2290                 } else if (tae.getType() == BgpRouterException.BGP_ERR_COMMON_FAILURE) {
2291                     LOG.debug("Starting the routesync for AS-ID started exception", tae);
2292                     startBgpRetry.errorOccured();
2293                     if (!startBgpRetry.shouldRetry()) {
2294                         doRouteSync = true;
2295                         replayDone = true;
2296                     }
2297                 } else {
2298                     LOG.error("Replay: startBgp() received exception type {}: ",
2299                             tae.getType(), tae);
2300                     startBgpRetry.errorOccured();
2301                 }
2302             } catch (Exception e) {
2303                 //not unusual. We may have restarted & BGP is already on
2304                 LOG.error("Replay:startBgp() received exception: ", e);
2305                 startBgpRetry.errorOccured();
2306             }
2307         } while (startBgpRetry.shouldRetry());
2308
2309         replaySucceded = replayDone;
2310
2311         startBgpCountersTask();
2312         startBgpAlarmsTask();
2313
2314         /*
2315          * commenting this due to a bug with QBGP. Will uncomment once QBGP fix is done.
2316          * This wont have any functional impacts
2317          */
2318         //try {
2319         //    br.delayEOR(delayEorSeconds);
2320         //} catch (TException | BgpRouterException e) {
2321         //    LOG.error("Replay: delayEOR() number of seconds to wait for EOR from ODL:", e);
2322         //}
2323
2324         BfdConfig bfdConfig = bgpUtil.getBfdConfig();
2325         if (bfdConfig != null) {
2326             if (bfdConfig.isBfdEnabled()) {
2327                 LOG.debug("Replaying bfd config min-rx {} min-tx {} detect-mul {} mhop {}",
2328                         bfdConfig.getMinRx(), bfdConfig.getMinTx(),
2329                         bfdConfig.getDetectMult(), bfdConfig.isMultihop());
2330                 try {
2331                     br.addBfd(bfdConfig.getDetectMult().intValue(), bfdConfig.getMinRx().intValue(),
2332                             bfdConfig.getMinTx().intValue(), bfdConfig.isMultihop());
2333                 } catch (TApplicationException tae) {
2334                     if (tae.getType() == BgpRouterException.BGP_ERR_PEER_EXISTS) {
2335                         LOG.debug("Replay:addBfd() received exception", tae);
2336                     } else {
2337                         LOG.error("Replay:addBfd() received exception", tae);
2338                     }
2339                 } catch (TException | BgpRouterException e) {
2340                     LOG.error("Replay:addBfd() received exception", e);
2341                 }
2342             }
2343         }
2344
2345         List<Neighbors> neighbors = config.getNeighborsContainer() == null ? null
2346                 : config.getNeighborsContainer().getNeighbors();
2347         if (neighbors != null) {
2348             LOG.error("configuring existing Neighbors present for replay total neighbors {}", neighbors.size());
2349             boolean neighborConfigReplayResult = replayNbrConfig(neighbors, br);
2350             if (neighborConfigReplayResult == false) {
2351                 replaySucceded = false;
2352             }
2353         } else {
2354             LOG.error("no Neighbors present for replay config ");
2355         }
2356
2357         Logging logging = config.getLogging();
2358         if (logging != null) {
2359             try {
2360                 br.setLogging(logging.getFile(), logging.getLevel());
2361             } catch (TException | BgpRouterException e) {
2362                 LOG.error("Replay:setLogging() received exception", e);
2363             }
2364         }
2365
2366         GracefulRestart gracefulRestart = config.getGracefulRestart();
2367         bgpGrRestartTime = ((gracefulRestart != null)
2368                 ? gracefulRestart.getStalepathTime().intValue() : bgpGrRestartTime);
2369         try {
2370             br.addGracefulRestart(bgpGrRestartTime);
2371         } catch (Exception e) {
2372             LOG.error("Replay:addGr() received exception: ", e);
2373         }
2374         List<Vrfs> vrfs = config.getVrfsContainer() == null ? null
2375                 : config.getVrfsContainer().getVrfs();
2376         if (vrfs == null) {
2377             vrfs = new ArrayList<>();
2378         }
2379         for (Vrfs vrf : vrfs) {
2380             for (AddressFamiliesVrf adf : vrf.getAddressFamiliesVrf()) {
2381                 try {
2382                     br.addVrf(BgpUtil.getLayerType(adf), vrf.getRd(), vrf.getImportRts(),
2383                             vrf.getExportRts(), adf.getAfi().toJava(), adf.getSafi().toJava());
2384                 } catch (TException | BgpRouterException e) {
2385                     LOG.error("Replay:addVrf() received exception", e);
2386                 }
2387             }
2388         }
2389
2390
2391         List<Networks> ln = config.getNetworksContainer() == null ? null
2392                 : config.getNetworksContainer().getNetworks();
2393         if (ln != null) {
2394             for (Networks net : ln) {
2395                 String rd = net.getRd();
2396                 String pfxlen = net.getPrefixLen();
2397                 String nh = net.getNexthop().getValue();
2398                 Long label = net.getLabel().toJava();
2399                 int lbl = label == null ? 0 : label.intValue();
2400                 int l3vni = net.getL3vni() == null ? 0 : net.getL3vni().intValue();
2401                 int l2vni = net.getL2vni() == null ? 0 : net.getL2vni().intValue();
2402                 if (rd == null && lbl > 0) {
2403                     //LU prefix is being deleted.
2404                     rd = Integer.toString(lbl);
2405                 }
2406
2407                 BgpControlPlaneType protocolType = net.getBgpControlPlaneType();
2408                 int ethernetTag = net.getEthtag().intValue();
2409                 String esi = net.getEsi();
2410                 String macaddress = net.getMacaddress();
2411                 EncapType encapType = net.getEncapType();
2412                 String routerMac = net.getRoutermac();
2413
2414                 try {
2415                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni,
2416                             BgpUtil.convertToThriftProtocolType(protocolType),
2417                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
2418                 } catch (TException | BgpRouterException e) {
2419                     LOG.error("Replay:addPfx() received exception", e);
2420                 }
2421             }
2422         }
2423
2424
2425         List<Multipath> multipaths = config.getMultipathContainer() == null ? null
2426                 : config.getMultipathContainer().getMultipath();
2427
2428         if (multipaths != null) {
2429             for (Multipath multipath : multipaths) {
2430                 if (multipath != null) {
2431                     af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
2432                     af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
2433
2434                     try {
2435                         if (multipath.isMultipathEnabled()) {
2436                             br.enableMultipath(afi, safi);
2437                         } else {
2438                             br.disableMultipath(afi, safi);
2439                         }
2440                     } catch (TException | BgpRouterException e) {
2441                         LOG.info("Replay:multipaths() received exception", e);
2442                     }
2443                 }
2444             }
2445         }
2446         List<VrfMaxpath> vrfMaxpaths = config.getVrfMaxpathContainer() == null ? null
2447                 : config.getVrfMaxpathContainer().getVrfMaxpath();
2448         if (vrfMaxpaths != null) {
2449             for (VrfMaxpath vrfMaxpath : vrfMaxpaths) {
2450                 try {
2451                     br.multipaths(vrfMaxpath.getRd(), vrfMaxpath.getMaxpaths().toJava());
2452                 } catch (TException | BgpRouterException e) {
2453                     LOG.info("Replay:vrfMaxPath() received exception", e);
2454                 }
2455             }
2456         }
2457
2458         //send End of Rib Marker to Qthriftd.
2459         final int numberOfEORRetries = 3;
2460         RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
2461         do {
2462             try {
2463                 br.sendEOR();
2464                 LOG.debug("Replay sendEOR() successful");
2465                 break;
2466             } catch (Exception e) {
2467                 eorRetry.errorOccured();
2468                 LOG.error("Replay:sedEOR() received exception:", e);
2469             }
2470         } while (eorRetry.shouldRetry());
2471
2472         if (doRouteSync) {
2473             LOG.debug("starting route sync for Thrift BGP_ERR_COMMON_FAILURE exception "
2474                       + "happened earlier");
2475             doRouteSync();
2476         }
2477
2478         return replaySucceded;
2479     }
2480
2481     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
2482         bgpUtil.update(iid, dto);
2483     }
2484
2485     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
2486         bgpUtil.delete(iid);
2487     }
2488
2489     public void startConfig(String bgpHost, int thriftPort) {
2490         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2491                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2492         InstanceIdentifier<ConfigServer> iid = iib.build();
2493         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
2494         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
2495                 .setPort((long) thriftPort).build();
2496         update(iid, dto);
2497     }
2498
2499     public void startBgp(long as, String routerId, int spt, boolean fbit) {
2500         IpAddress rid = routerId == null ? null : IpAddressBuilder.getDefaultInstance(routerId);
2501         Long staleTime = (long) spt;
2502         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2503                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2504         InstanceIdentifier<AsId> iid = iib.build();
2505         AsId dto = new AsIdBuilder().setLocalAs(as)
2506                 .setRouterId(rid)
2507                 .setStalepathTime(staleTime)
2508                 .setAnnounceFbit(fbit).build();
2509         update(iid, dto);
2510     }
2511
2512     public void startBfd(long detectMult, long minRx, long minTx, boolean multiHop) {
2513         InstanceIdentifier.InstanceIdentifierBuilder<BfdConfig> iib =
2514                 InstanceIdentifier.builder(BfdConfig.class);
2515         InstanceIdentifier<BfdConfig> iid = iib.build();
2516         BfdConfig dto = new BfdConfigBuilder()
2517                 .setBfdEnabled(true)
2518                 .setMultihop(multiHop)
2519                 .setMinRx(minRx)
2520                 .setMinTx(minTx)
2521                 .setDetectMult(detectMult)
2522                 .build();
2523         update(iid, dto);
2524     }
2525
2526     public void addDcgwTep(String dcgwIp, String tepIp) {
2527         InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2528                 InstanceIdentifier.builder(Bgp.class)
2529                         .child(DcgwTepList.class)
2530                         .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2531         InstanceIdentifier<DcgwTep> iid = iib.build();
2532         ArrayList<String> tepList = new ArrayList<String>();
2533         tepList.add(tepIp);
2534         DcgwTep dto = new DcgwTepBuilder().setDcGwIp(dcgwIp).setTepIps(tepList)
2535                 .build();
2536         update(iid, dto);
2537         bgpUtil.removeOrUpdateLBGroups(tepIp,NwConstants.MOD_FLOW);
2538     }
2539
2540     public void addLogging(String fileName, String logLevel) {
2541         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2542                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2543         InstanceIdentifier<Logging> iid = iib.build();
2544         Logging dto = new LoggingBuilder().setFile(fileName)
2545                 .setLevel(logLevel).build();
2546         update(iid, dto);
2547     }
2548
2549     public void addGracefulRestart(int staleTime) {
2550         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2551                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
2552         InstanceIdentifier<GracefulRestart> iid = iib.build();
2553         GracefulRestart dto = new GracefulRestartBuilder()
2554                 .setStalepathTime((long) staleTime).build();
2555         update(iid, dto);
2556     }
2557
2558     public void addNeighbor(
2559             String nbrIp, long remoteAs, @Nullable final TcpMd5SignaturePasswordType md5Secret) {
2560         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2561         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2562                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2563                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2564         InstanceIdentifier<Neighbors> iid = iib.build();
2565         TcpSecurityOption tcpSecOption = null;
2566         if (md5Secret != null) {
2567             tcpSecOption = new TcpMd5SignatureOptionBuilder().setTcpMd5SignaturePassword(md5Secret).build();
2568         } // else let tcpSecOption be null
2569         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
2570                 .setRemoteAs(remoteAs).setTcpSecurityOption(tcpSecOption).build();
2571         update(iid, dto);
2572     } // public addNeighbor(nbrIp, remoteAs, md5Secret)
2573
2574     public void addUpdateSource(String nbrIp, String srcIp) {
2575         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2576         Ipv4Address srcAddr = new Ipv4Address(srcIp);
2577         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2578                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2579                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2580                         .child(UpdateSource.class);
2581         InstanceIdentifier<UpdateSource> iid = iib.build();
2582         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
2583                 .setSourceIp(srcAddr).build();
2584         update(iid, dto);
2585     }
2586
2587     public void addEbgpMultihop(String nbrIp, int hops) {
2588         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2589         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2590                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2591                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2592                         .child(EbgpMultihop.class);
2593         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2594         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
2595                 .setNhops((long) hops).build();
2596         update(iid, dto);
2597     }
2598
2599     public void addAddressFamily(String nbrIp, int afi, int safi) {
2600         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2601         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2602                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2603                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2604                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2605         InstanceIdentifier<AddressFamilies> iid = iib.build();
2606         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
2607                 .setAfi((long) afi).setSafi((long) safi).build();
2608         update(iid, dto);
2609     }
2610
2611     public void addPrefix(String rd, String macAddress, String pfx, List<String> nhList,
2612               VrfEntry.EncapType encapType, Uint32 lbl, Uint32 l3vni, Uint32 l2vni, String gatewayMac) {
2613         for (String nh : nhList) {
2614             Ipv4Address nexthop = nh != null ? new Ipv4Address(nh) : null;
2615             Uint32 label = lbl;
2616             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
2617                     .child(NetworksContainer.class)
2618                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
2619             NetworksBuilder networksBuilder = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
2620                                                 .setLabel(label).setEthtag(BgpConstants.DEFAULT_ETH_TAG);
2621             buildVpnEncapSpecificInfo(networksBuilder, encapType, label, l3vni, l2vni, macAddress, gatewayMac);
2622             update(iid, networksBuilder.build());
2623         }
2624     }
2625
2626     private static void buildVpnEncapSpecificInfo(NetworksBuilder builder, VrfEntry.EncapType encapType, Uint32 label,
2627                                                   Uint32 l3vni, Uint32 l2vni, String macAddress, String gatewayMac) {
2628         if (encapType.equals(VrfEntry.EncapType.Mplsgre)) {
2629             builder.setLabel(label).setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLL3VPN)
2630                     .setEncapType(EncapType.GRE);
2631         } else {
2632             builder.setL3vni(l3vni).setL2vni(l2vni).setMacaddress(macAddress).setRoutermac(gatewayMac)
2633                     .setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLEVPN).setEncapType(EncapType.VXLAN);
2634         }
2635     }
2636
2637     // TODO: add LayerType as arg - supports command
2638     public void addVrf(String rd, List<String> irts, List<String> erts, AddressFamily addressFamily) {
2639         Vrfs vrf = bgpUtil.getVrfFromRd(rd);
2640         List<AddressFamiliesVrf> adfList = new ArrayList<>(1);
2641         if (vrf != null) {
2642             adfList = vrf.getAddressFamiliesVrf();
2643         }
2644         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2645         if (addressFamily.equals(AddressFamily.IPV4)) {
2646             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2647             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2648         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2649             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2650             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2651         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2652             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2653             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2654         }
2655         AddressFamiliesVrf adf = adfBuilder.build();
2656         adfList.add(adf);
2657         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib = InstanceIdentifier.builder(Bgp.class)
2658                 .child(VrfsContainer.class)
2659                 .child(Vrfs.class, new VrfsKey(rd));
2660         InstanceIdentifier<Vrfs> iid = iib.build();
2661         Vrfs dto = new VrfsBuilder().setRd(rd).setImportRts(irts)
2662             .setExportRts(erts).setAddressFamiliesVrf(adfList).build();
2663
2664         List<AddressFamiliesVrf> listAdFamilies = mapNewAdFamily.get(rd);
2665         if (listAdFamilies != null) {
2666             listAdFamilies.add(adf);
2667         } else {
2668             mapNewAdFamily.put(rd, adfList);
2669         }
2670
2671         try {
2672             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
2673         } catch (TransactionCommitFailedException e) {
2674             LOG.error("Error adding VRF to datastore", e);
2675             throw new RuntimeException(e);
2676         }
2677
2678         // enable multipath by default in all VRFs
2679         setMultipaths(rd, BgpConstants.BGP_DEFAULT_MULTIPATH);
2680     }
2681
2682     public void stopConfig() {
2683         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2684                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2685         InstanceIdentifier<ConfigServer> iid = iib.build();
2686         delete(iid);
2687     }
2688
2689     public void stopBgp() {
2690         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2691                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2692         InstanceIdentifier<AsId> iid = iib.build();
2693         delete(iid);
2694     }
2695
2696     public void stopBfd() {
2697         InstanceIdentifier.InstanceIdentifierBuilder<BfdConfig> iib =
2698                 InstanceIdentifier.builder(BfdConfig.class);
2699         InstanceIdentifier<BfdConfig> iid = iib.build();
2700         delete(iid);
2701     }
2702
2703     public void delDcgwTep(String dcgwIp, String tepIp) {
2704         if (tepIp == null) {
2705             InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2706                     InstanceIdentifier.builder(Bgp.class)
2707                             .child(DcgwTepList.class)
2708                             .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2709             InstanceIdentifier<DcgwTep> iid = iib.build();
2710             delete(iid);
2711         } else {
2712             InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2713                     InstanceIdentifier.builder(Bgp.class)
2714                             .child(DcgwTepList.class)
2715                             .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2716             InstanceIdentifier<DcgwTep> iid = iib.build();
2717             List<String> tepIpList = bgpUtil.getDcgwTepConfig(dcgwIp);
2718             if (tepIpList == null) {
2719                 LOG.error("No Tep IP configured for DCGW {} on deleting the dcgwtep", dcgwIp);
2720                 return;
2721             }
2722             List<String> newTepIpList = new ArrayList<String>();
2723             tepIpList.forEach(tep -> {
2724                 if (!tep.equals(tepIp)) {
2725                     newTepIpList.add(tep);
2726                 }
2727             });
2728             DcgwTep dto = new DcgwTepBuilder().setDcGwIp(dcgwIp).setTepIps(newTepIpList)
2729                     .build();
2730             try {
2731                 SingleTransactionDataBroker.syncWrite(dataBroker,
2732                         LogicalDatastoreType.CONFIGURATION, iid, dto);
2733             } catch (TransactionCommitFailedException e) {
2734                 LOG.error("delDcgwTep: Error deleting DCGW Tep", e);
2735                 throw new RuntimeException(e);
2736             }
2737         }
2738     }
2739
2740     public void delLogging() {
2741         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2742                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2743         InstanceIdentifier<Logging> iid = iib.build();
2744         delete(iid);
2745     }
2746
2747     public void delGracefulRestart() {
2748         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2749                 InstanceIdentifier.builder(Bgp.class)
2750                         .child(GracefulRestart.class);
2751         InstanceIdentifier<GracefulRestart> iid = iib.build();
2752         delete(iid);
2753     }
2754
2755     public void delNeighbor(String nbrIp) {
2756         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2757         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2758                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2759                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2760         InstanceIdentifier<Neighbors> iid = iib.build();
2761         delete(iid);
2762     }
2763
2764     public void delUpdateSource(String nbrIp) {
2765         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2766         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2767                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2768                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2769                         .child(UpdateSource.class);
2770         InstanceIdentifier<UpdateSource> iid = iib.build();
2771         delete(iid);
2772     }
2773
2774     public void delEbgpMultihop(String nbrIp) {
2775         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2776         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2777                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2778                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2779                         .child(EbgpMultihop.class);
2780         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2781         delete(iid);
2782     }
2783
2784     public void delAddressFamily(String nbrIp, int afi, int safi) {
2785         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2786         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2787                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2788                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2789                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2790         InstanceIdentifier<AddressFamilies> iid = iib.build();
2791         delete(iid);
2792     }
2793
2794     public void delPrefix(String rd, String pfx) {
2795         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
2796                 InstanceIdentifier.builder(Bgp.class).child(NetworksContainer.class)
2797                         .child(Networks.class, new NetworksKey(pfx, rd));
2798         InstanceIdentifier<Networks> iid = iib.build();
2799         delete(iid);
2800     }
2801
2802     public boolean delVrf(String rd, AddressFamily addressFamily) {
2803         if (addressFamily == null) {
2804             LOG.error("delVrf: vrf {}, addressFamily invalid", rd);
2805             return false;
2806         }
2807         delMultipaths(rd);
2808         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2809         if (addressFamily.equals(AddressFamily.IPV4)) {
2810             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2811             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2812         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2813             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2814             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2815         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2816             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2817             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2818         }
2819         LOG.debug("delVrf: Received Delete VRF : rd:{}, address family: {} {}", rd,
2820                 adfBuilder.getAfi(), adfBuilder.getSafi());
2821
2822         Vrfs vrfOriginal = bgpUtil.getVrfFromRd(rd);
2823         if (vrfOriginal == null) {
2824             LOG.error("delVrf: no vrf with existing rd {}. step aborted", rd);
2825             return false;
2826         }
2827
2828         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2829                 InstanceIdentifier.builder(Bgp.class).child(VrfsContainer.class)
2830                         .child(Vrfs.class, new VrfsKey(rd));
2831
2832         InstanceIdentifier<Vrfs> iid = iib.build();
2833
2834         @SuppressWarnings("static-access")
2835         InstanceIdentifier<Bgp> iid6 =  iid.builder(Bgp.class).build()
2836                 .child(MultipathContainer.class)
2837                 .child(Multipath.class, new MultipathKey(adfBuilder.getAfi(), adfBuilder.getSafi())).create(Bgp.class);
2838         InstanceIdentifierBuilder<Vrfs> iib3 =
2839                 iid6.child(VrfsContainer.class).child(Vrfs.class, new VrfsKey(rd)).builder();
2840         InstanceIdentifier<Vrfs> iidFinal = iib3.build();
2841
2842         //** update or delete the vrfs with the rest of AddressFamilies already present in the last list
2843         AddressFamiliesVrf adfToDel = adfBuilder.build();
2844         List<AddressFamiliesVrf> adfListOriginal = new ArrayList<>(vrfOriginal.nonnullAddressFamiliesVrf());
2845         List<AddressFamiliesVrf> adfListToRemoveFromOriginal = new ArrayList<>();
2846         adfListOriginal.forEach(adf -> {
2847             if (adf.equals(adfToDel)) {
2848                 adfListToRemoveFromOriginal.add(adfToDel);
2849                 return;
2850             }
2851         });
2852         for (AddressFamiliesVrf adfToRemove : adfListToRemoveFromOriginal) {
2853             adfListOriginal.remove(adfToRemove);
2854             try {
2855                 SingleTransactionDataBroker.syncWrite(dataBroker,
2856                         LogicalDatastoreType.CONFIGURATION, iid, vrfOriginal);
2857             } catch (TransactionCommitFailedException e) {
2858                 LOG.error("delVrf: Error updating VRF to datastore", e);
2859                 throw new RuntimeException(e);
2860             }
2861         }
2862         if (adfListOriginal.isEmpty()) {
2863             LOG.debug("delVrf: delete iid: {}", iidFinal);
2864             delete(iidFinal);
2865             return true;
2866         }
2867         // not all is removed
2868         return false;
2869     }
2870
2871     public void setMultipathStatus(af_afi afi, af_safi safi, boolean enable) {
2872         long lafi = afi.getValue();
2873         long lsafi = safi.getValue();
2874
2875         InstanceIdentifier.InstanceIdentifierBuilder<Multipath> iib =
2876                 InstanceIdentifier
2877                         .builder(Bgp.class).child(MultipathContainer.class)
2878                         .child(Multipath.class,
2879                                 new MultipathKey(Long.valueOf(afi.getValue()), Long.valueOf(safi.getValue())));
2880
2881         Multipath dto = new MultipathBuilder().setAfi(lafi).setSafi(lsafi).setMultipathEnabled(enable).build();
2882         update(iib.build(), dto);
2883     }
2884
2885     public void setMultipaths(String rd, int maxpath) {
2886         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
2887                 InstanceIdentifier
2888                         .builder(Bgp.class).child(VrfMaxpathContainer.class)
2889                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
2890
2891         VrfMaxpath dto = new VrfMaxpathBuilder().setRd(rd).setMaxpaths(maxpath).build();
2892         update(iib.build(), dto);
2893     }
2894
2895     public void delMultipaths(String rd) {
2896         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
2897                 InstanceIdentifier.builder(Bgp.class).child(VrfMaxpathContainer.class)
2898                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
2899         InstanceIdentifier<VrfMaxpath> iid = iib.build();
2900         delete(iid);
2901     }
2902
2903     /*
2904     * Remove Stale Marked Routes after timer expiry.
2905     */
2906     private class RouteCleanup implements Callable<Integer> {
2907
2908         @Override
2909         public Integer call() {
2910             totalCleared = 0;
2911             try {
2912                 if (staledFibEntriesMap.isEmpty()) {
2913                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
2914                 } else {
2915                     for (String rd : staledFibEntriesMap.keySet()) {
2916                         if (Thread.interrupted()) {
2917                             return 0;
2918                         }
2919                         Map<String, Uint32> map = staledFibEntriesMap.get(rd);
2920                         if (map != null) {
2921                             for (String key : map.keySet()) {
2922                                 if (Thread.interrupted()) {
2923                                     return 0;
2924                                 }
2925                                 String prefix = extractPrefix(key);
2926                                 String nextHop = extractNextHop(key);
2927                                 totalCleared++;
2928                                 LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}, nextHop:{}",
2929                                         rd, prefix, nextHop);
2930                                 fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix, nextHop);
2931                             }
2932                         }
2933                     }
2934                 }
2935             } finally {
2936                 staledFibEntriesMap.clear();
2937             }
2938             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
2939             return 0;
2940         }
2941     }
2942
2943     /*
2944      * BGP restart scenario, ODL-BGP manager was/is running.
2945      * On re-sync notification, Get a copy of FIB database.
2946      */
2947     public void createStaleFibMap() {
2948         totalStaledCount = 0;
2949         try {
2950             staledFibEntriesMap.clear();
2951             fibDSWriter.clearFibMap();
2952             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2953
2954             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2955                     LogicalDatastoreType.CONFIGURATION, id);
2956             if (fibEntries.isPresent()) {
2957                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
2958                 for (VrfTables vrfTable : staleVrfTables) {
2959                     Map<String, Uint32> staleFibEntMap = new HashMap<>();
2960                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2961                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2962                             //Stale marking and cleanup is only meant for the routes learned through BGP.
2963                             continue;
2964                         }
2965                         if (Thread.interrupted()) {
2966                             break;
2967                         }
2968                         totalStaledCount++;
2969                         //Create MAP from staleVrfTables.
2970                         vrfEntry.getRoutePaths()
2971                                 .forEach(
2972                                     routePath -> {
2973                                         staleFibEntMap.put(
2974                                                 appendNextHopToPrefix(vrfEntry.getDestPrefix(),
2975                                                         routePath.getNexthopAddress()), routePath.getLabel());
2976                                         fibDSWriter.addEntryToFibMap(
2977                                                 vrfTable.getRouteDistinguisher(),  vrfEntry.getDestPrefix(),
2978                                                 routePath.getNexthopAddress());
2979
2980                                     });
2981                     }
2982                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), staleFibEntMap);
2983                 }
2984             } else {
2985                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
2986             }
2987         } catch (InterruptedException | ExecutionException e) {
2988             LOG.error("createStaleFibMap:: error ", e);
2989         }
2990         LOG.error("created {} staled entries ", totalStaledCount);
2991     }
2992
2993     /*
2994      * BGP config remove scenario, Need to remove all the
2995      * external routes from FIB.
2996      */
2997     public void deleteExternalFibRoutes() {
2998         totalExternalRoutes = 0;
2999         totalExternalMacRoutes = 0;
3000         try {
3001             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
3002
3003             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
3004                     LogicalDatastoreType.CONFIGURATION, id);
3005             if (fibEntries.isPresent()) {
3006                 if (fibEntries.get().getVrfTables() == null) {
3007                     LOG.error("deleteExternalFibRoutes::getVrfTables is null");
3008                     return;
3009                 }
3010                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
3011                 for (VrfTables vrfTable : staleVrfTables) {
3012                     String rd = vrfTable.getRouteDistinguisher();
3013                     if (vrfTable.getVrfEntry() != null) {
3014                         for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
3015                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
3016                                 //route cleanup is only meant for the routes learned through BGP.
3017                                 continue;
3018                             }
3019                             totalExternalRoutes++;
3020                             fibDSWriter.removeFibEntryFromDS(rd, vrfEntry.getDestPrefix());
3021                         }
3022                     } else if (vrfTable.getMacVrfEntry() != null) {
3023                         for (MacVrfEntry macEntry : vrfTable.getMacVrfEntry()) {
3024                             if (RouteOrigin.value(macEntry.getOrigin()) != RouteOrigin.BGP) {
3025                                 //route cleanup is only meant for the routes learned through BGP.
3026                                 continue;
3027                             }
3028                             totalExternalMacRoutes++;
3029                             fibDSWriter.removeMacEntryFromDS(rd, macEntry.getMac());
3030                         }
3031                     }
3032                 }
3033             } else {
3034                 LOG.error("deleteExternalFibRoutes:: FIBentries.class is not present");
3035             }
3036         } catch (InterruptedException | ExecutionException e) {
3037             LOG.error("deleteExternalFibRoutes:: error ", e);
3038         }
3039         LOG.debug("deleted {} fib entries {} mac entries", totalExternalRoutes, totalExternalMacRoutes);
3040     }
3041
3042     public boolean addToRt2TepMap(String rd, String tepIp, String mac, Uint32 l2vni) {
3043         boolean isFirstMacUpdateFromTep = false;
3044         if (rt2TepMap.containsKey(rd)) {
3045             if (rt2TepMap.get(rd).containsKey(tepIp)) {
3046                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and tep-ip {}. No Elan DS write required",
3047                         mac, l2vni, rd, tepIp);
3048                 rt2TepMap.get(rd).get(tepIp).put(mac, l2vni);
3049             } else {
3050                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and new tep-ip {}",
3051                         mac, l2vni, rd, tepIp);
3052                 isFirstMacUpdateFromTep = true;
3053                 Map<String, Uint32> macList = new HashMap<>();
3054                 macList.put(mac, l2vni);
3055                 rt2TepMap.get(rd).put(tepIp, macList);
3056             }
3057         } else {
3058             LOG.debug("RT2 with mac {} l2vni {} from new rd {} and tep ip {}",
3059                     mac, l2vni, rd, tepIp);
3060             isFirstMacUpdateFromTep = true;
3061             Map<String, Uint32> macList = new HashMap<>();
3062             macList.put(mac, l2vni);
3063             Map<String, Map<String, Uint32>> tepIpMacMap = new HashMap<>();
3064             tepIpMacMap.put(tepIp, macList);
3065             rt2TepMap.put(rd, tepIpMacMap);
3066         }
3067         return isFirstMacUpdateFromTep;
3068     }
3069
3070     public boolean deleteFromRt2TepMap(String rd, String tepIp, String mac) {
3071         boolean isLastMacUpdateFromTep = false;
3072         LOG.debug("RT2 withdraw with rd {} mac {} tep-ip {} ", rd, mac, tepIp);
3073         if (rt2TepMap.containsKey(rd)) {
3074             if (rt2TepMap.get(rd).containsKey(tepIp)) {
3075                 if (rt2TepMap.get(rd).get(tepIp).containsKey(mac)) {
3076                     LOG.debug("RT2 Withdraw : Removing the mac {} from Map", mac);
3077                     rt2TepMap.get(rd).get(tepIp).remove(mac);
3078                     if (rt2TepMap.get(rd).get(tepIp).isEmpty()) {
3079                         isLastMacUpdateFromTep = true;
3080                         LOG.debug("RT2 Withdraw : Removing the tep-ip {} from Map", tepIp);
3081                         rt2TepMap.get(rd).remove(tepIp);
3082                         if (rt2TepMap.get(rd).isEmpty()) {
3083                             LOG.debug("RT2 Withdraw : Removing the rd {} from Map", rd);
3084                             rt2TepMap.remove(rd);
3085                         }
3086                     }
3087                 }
3088             }
3089         }
3090         return isLastMacUpdateFromTep;
3091     }
3092
3093     public Collection<String> getTepIPs(String rd) {
3094         final Map<String, Map<String, Uint32>> tepIpMap = rt2TepMap.get(rd);
3095         return tepIpMap != null ? tepIpMap.keySet() : Collections.emptyList();
3096     }
3097
3098     public boolean isBgpConnected() {
3099         return (bgpRouter == null) ? false : bgpRouter.isBgpConnected();
3100     }
3101
3102     public long getLastConnectedTS() {
3103         return (bgpRouter == null) ? 0 : bgpRouter.getLastConnectedTS();
3104     }
3105
3106     public long getConnectTS() {
3107         return (bgpRouter == null) ? 0 : bgpRouter.getConnectTS();
3108     }
3109
3110     public long getStartTS() {
3111         return (bgpRouter == null) ? 0 : bgpRouter.getStartTS();
3112     }
3113
3114     public TTransport getTransport() {
3115         return bgpRouter.getTransport();
3116     }
3117
3118     public int getTotalStaledCount() {
3119         return totalStaledCount;
3120     }
3121
3122     public int getTotalCleared() {
3123         return totalCleared;
3124     }
3125
3126     public static List<Neighbors> getNbrList() {
3127         return nbrList;
3128     }
3129
3130     public BgpCounters getBgpCounters() {
3131         return bgpCountersReference.get();
3132     }
3133
3134     private void startBgpCountersTask() {
3135         if (getBgpCounters() == null && bgpCountersReference.compareAndSet(null,
3136                 new BgpCounters(getBgpSdncMipIp(), metricProvider))) {
3137             bgpCountersTask = executor.scheduleAtFixedRate(bgpCountersReference.get(), 0, 120 * 1000,
3138                     TimeUnit.MILLISECONDS);
3139             LOG.info("Bgp Counters task scheduled for every two minutes.");
3140
3141             bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
3142         }
3143     }
3144
3145     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3146             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3147     private void stopBgpCountersTask() {
3148         final BgpCounters bgpCounters = bgpCountersReference.getAndSet(null);
3149         if (bgpCounters != null) {
3150             bgpCountersTask.cancel(true);
3151             bgpCounters.close();
3152         }
3153     }
3154
3155     private void startBgpAlarmsTask() {
3156         if (getBgpAlarms() == null && bgpAlarmsReference.compareAndSet(null, new BgpAlarms(this))) {
3157             bgpAlarmsReference.get().init();
3158             bgpAlarmsTask = executor.scheduleAtFixedRate(bgpAlarmsReference.get(), 0, 60 * 1000, TimeUnit.MILLISECONDS);
3159             LOG.info("Bgp Alarms task scheduled for every minute.");
3160         } else {
3161             LOG.trace("Bgp Alarms task already scheduled for every minute.");
3162         }
3163     }
3164
3165     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3166             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3167     private void stopBgpAlarmsTask() {
3168         final BgpAlarms bgpAlarms = bgpAlarmsReference.getAndSet(null);
3169         if (bgpAlarms != null) {
3170             bgpAlarmsTask.cancel(true);
3171             bgpAlarms.close();
3172         }
3173     }
3174
3175     public BgpAlarms getBgpAlarms() {
3176         return bgpAlarmsReference.get();
3177     }
3178
3179     public void getPeerStatus(String nbrIp, long nbrAsNum) throws
3180             BgpRouterException, TException {
3181         bgpRouter.getPeerStatus(nbrIp, nbrAsNum);
3182     }
3183
3184     private static String appendNextHopToPrefix(String prefix, String nextHop) {
3185         return prefix + ":" + nextHop;
3186     }
3187
3188     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3189             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3190     private static String extractPrefix(String prefixNextHop) {
3191         return prefixNextHop.split(":")[0];
3192     }
3193
3194     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3195             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3196     private static String extractNextHop(String prefixNextHop) {
3197         return prefixNextHop.split(":")[1];
3198     }
3199
3200     private static String extractMd5Secret(final Neighbors val) {
3201         String md5Secret = null;
3202         TcpSecurityOption tcpSecOpt = val.getTcpSecurityOption();
3203         if (tcpSecOpt != null) {
3204             if (tcpSecOpt instanceof TcpMd5SignatureOption) {
3205                 md5Secret = ((TcpMd5SignatureOption) tcpSecOpt).getTcpMd5SignaturePassword().getValue();
3206             } else { // unknown TcpSecurityOption
3207                 LOG.debug("neighbors  Ignored unknown tcp-security-option of peer {}", val.getAddress().getValue());
3208             }
3209         }
3210         return md5Secret;
3211     } // private method extractMd5Secret
3212
3213     @SuppressWarnings("checkstyle:IllegalCatch")
3214     @Override
3215     public ListenableFuture<RpcResult<InitiateEorOutput>> initiateEor(InitiateEorInput input) {
3216         boolean returnError = false;
3217         String msg = null;
3218         String neighborIp = null;
3219         if (!isBGPEntityOwner()) {
3220             msg = String.format("RPC triggered in Non-EoS Owner");
3221             return Futures.immediateFuture(
3222                     RpcResultBuilder.<InitiateEorOutput>failed().withError(RpcError.ErrorType.APPLICATION,
3223                             msg).build());
3224         }
3225         if (input == null) {
3226             msg = String.format("BGP invalid input for EoR");
3227             LOG.error("Error : {}", msg);
3228             returnError = true;
3229         } else {
3230             neighborIp = input.getNeighborIp();
3231         }
3232         if (eorSupressedDuetoUpgradeFlag.get() == false) {
3233             msg = String.format("EoR triggerd by RBU-RPC call before replay"
3234                     + "of BGP configuration (or) BGP not restarted");
3235             LOG.error("Error : {}", msg);
3236         }
3237         if ("ALL".compareToIgnoreCase(neighborIp) == 0) {
3238             //send EoR for all the neighbor
3239             LOG.error("EoR trigger received to ALL neighbors");
3240             final int numberOfEORRetries = 3;
3241             RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
3242             do {
3243                 try {
3244                     BgpRouter br = bgpRouter;
3245                     br.sendEOR();
3246                     LOG.debug("RPC: sendEOR {} successful", br);
3247                     break;
3248                 } catch (Exception e) {
3249                     eorRetry.errorOccured();
3250                     LOG.error("Replay:sedEOR() received exception:", e);
3251                 }
3252             } while (eorRetry.shouldRetry());
3253             eorSupressedDuetoUpgradeFlag.set(false);
3254         } else if (InetAddresses.isInetAddress(neighborIp)) {
3255             //send EoR for only one neighbor
3256             msg = String.format("Inidividual neighbors EoR is not supported");
3257             LOG.warn("Error : {}", msg);
3258             returnError = true;
3259         } else {
3260             //error
3261             msg = String.format("RPC: initiateEor: Invalid input ");
3262             LOG.warn("Error : {}", msg);
3263             returnError = true;
3264         }
3265         if (returnError) {
3266             return Futures.immediateFuture(
3267                     RpcResultBuilder.<InitiateEorOutput>failed().withError(RpcError.ErrorType.APPLICATION,
3268                             msg).build());
3269         }
3270         InitiateEorOutput initiateEorOutput =
3271                 new InitiateEorOutputBuilder().setRetVal(0L).build();
3272         return Futures.immediateFuture(RpcResultBuilder.<InitiateEorOutput>success()
3273                 .withResult(initiateEorOutput).build());
3274     }
3275 }