bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import static org.opendaylight.netvirt.bgpmanager.oam.BgpConstants.HISTORY_LIMIT;
  11 import static org.opendaylight.netvirt.bgpmanager.oam.BgpConstants.HISTORY_THRESHOLD;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.base.Preconditions;
  15 import com.google.common.net.InetAddresses;
  16 import com.google.common.util.concurrent.Futures;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18 import com.google.common.util.concurrent.ThreadFactoryBuilder;
  19 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  20 import io.netty.util.concurrent.GlobalEventExecutor;
  21 import java.lang.reflect.Constructor;
  22 import java.lang.reflect.InvocationTargetException;
  23 import java.net.InetAddress;
  24 import java.net.InetSocketAddress;
  25 import java.net.NetworkInterface;
  26 import java.net.SocketException;
  27 import java.util.ArrayList;
  28 import java.util.Collection;
  29 import java.util.Collections;
  30 import java.util.Enumeration;
  31 import java.util.HashMap;
  32 import java.util.Iterator;
  33 import java.util.List;
  34 import java.util.Map;
  35 import java.util.Objects;
  36 import java.util.Timer;
  37 import java.util.concurrent.Callable;
  38 import java.util.concurrent.ConcurrentHashMap;
  39 import java.util.concurrent.CountDownLatch;
  40 import java.util.concurrent.ExecutionException;
  41 import java.util.concurrent.Executors;
  42 import java.util.concurrent.Future;
  43 import java.util.concurrent.ScheduledExecutorService;
  44 import java.util.concurrent.ScheduledFuture;
  45 import java.util.concurrent.TimeUnit;
  46 import java.util.concurrent.TimeoutException;
  47 import java.util.concurrent.atomic.AtomicBoolean;
  48 import java.util.concurrent.atomic.AtomicInteger;
  49 import java.util.concurrent.atomic.AtomicReference;
  50 import javax.annotation.PreDestroy;
  51 import javax.inject.Inject;
  52 import javax.inject.Singleton;
  53 import org.apache.thrift.TApplicationException;
  54 import org.apache.thrift.TException;
  55 import org.apache.thrift.transport.TTransport;
  56 import org.eclipse.jdt.annotation.Nullable;
  57 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  58 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  59 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  60 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  61 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  62 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  63 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  64 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  65 import org.opendaylight.genius.mdsalutil.NwConstants;
  66 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  67 import org.opendaylight.infrautils.metrics.MetricProvider;
  68 import org.opendaylight.mdsal.eos.binding.api.Entity;
  69 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipCandidateRegistration;
  70 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipListenerRegistration;
  71 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  72 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  73 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  74 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  75 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  76 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  77 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  78 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  79 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  80 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  81 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  82 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  83 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  84 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  85 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  86 import org.opendaylight.netvirt.bgpmanager.thrift.gen.protocol_type;
  87 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  88 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  89 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  90 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  91 import org.opendaylight.ovsdb.utils.mdsal.utils.TransactionHistory;
  92 import org.opendaylight.ovsdb.utils.mdsal.utils.TransactionType;
  93 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebfd.rev190219.BfdConfig;
  94 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebfd.rev190219.BfdConfigBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.AddressFamily;
  96 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  97 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
  98 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EbgpService;
  99 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EncapType;
 100 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorInput;
 101 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorOutput;
 102 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorOutputBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.TcpMd5SignaturePasswordType;
 104 //import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.*;
 105 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
 106 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
 108 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.DcgwTepList;
 110 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
 111 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
 113 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathContainer;
 115 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsContainer;
 116 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksContainer;
 117 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathContainer;
 118 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsContainer;
 119 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTep;
 120 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTepBuilder;
 121 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTepKey;
 122 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.Multipath;
 123 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.MultipathBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.MultipathKey;
 125 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.Neighbors;
 126 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.NeighborsBuilder;
 127 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.NeighborsKey;
 128 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamilies;
 129 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamiliesBuilder;
 130 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamiliesKey;
 131 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.EbgpMultihop;
 132 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.EbgpMultihopBuilder;
 133 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.UpdateSource;
 134 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.UpdateSourceBuilder;
 135 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.Networks;
 136 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.NetworksBuilder;
 137 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.NetworksKey;
 138 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpath;
 139 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpathBuilder;
 140 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpathKey;
 141 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.Vrfs;
 142 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.VrfsBuilder;
 143 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.VrfsKey;
 144 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.vrfs.AddressFamiliesVrf;
 145 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.vrfs.AddressFamiliesVrfBuilder;
 146 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.TcpSecurityOption;
 147 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOption;
 148 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOptionBuilder;
 149 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
 150 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
 151 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
 152 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 153 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 154 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.macvrfentries.MacVrfEntry;
 155 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 156 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 157 import org.opendaylight.yangtools.yang.binding.DataObject;
 158 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 159 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 160 import org.opendaylight.yangtools.yang.common.RpcError;
 161 import org.opendaylight.yangtools.yang.common.RpcResult;
 162 import org.opendaylight.yangtools.yang.common.RpcResultBuilder;
 163 import org.opendaylight.yangtools.yang.common.Uint32;
 164 import org.osgi.framework.BundleContext;
 165 import org.osgi.util.tracker.ServiceTracker;
 166 import org.slf4j.Logger;
 167 import org.slf4j.LoggerFactory;
 168
 169 @Singleton
 170 public class BgpConfigurationManager implements EbgpService {
 171     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 172
 173     // to have stale FIB map (RD, Prefix)
 174     //  number of seconds wait for route sync-up between ODL and BGP
 175     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 176     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 177     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 178     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 179     private static final String DEF_UPORT = "6644";
 180     private static final String DEF_CHOST = "255.255.255.255"; // Invalid Host IP
 181     private static final String DEF_CPORT = "7644";
 182     private static final String DEF_BGP_SDNC_MIP = "127.0.0.1";
 183     //vpnservice.bgp.thrift.bgp.mip is the MIP present with ODL. Here we open 6644 port
 184     private static final String BGP_SDNC_MIP = "vpnservice.bgp.thrift.bgp.mip";
 185     private static final String BGP_GR_RESTART_TIMER_PROPERTY = "vpnservice.bgp.gr.timer";
 186     private static final String BGP_KA_TIMER_PROPERTY = "vpnservice.bgp.ka.timer";
 187     private static final String BGP_HOLD_TIMER_PROPERTY = "vpnservice.bgp.hold.timer";
 188     private static final String BGP_EOR_DELAY_PROPERTY = "vpnservice.bgp.eordelay";
 189     private static final int DEF_BGP_KA_TIME = 60;
 190     private static final int DEF_BGP_HOLD_TIME = 180;
 191     private static final int DEF_BGP_GR_TIME = 4000;
 192     private static final int RESTART_DEFAULT_GR = 90;
 193     private static final int DS_RETRY_COUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
 194     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
 195     private static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 196     private static final String BGP_EOR_DELAY = "vpnservice.bgp.eordelay";
 197     private static final String DEF_BGP_EOR_DELAY = "1800";
 198     private static final String BGP_ENTITY_NAME = "bgp";
 199     private static final String ADD_WARN = "Config store updated; undo with Delete if needed.";
 200     private static final String DEL_WARN = "Config store updated; undo with Add if needed.";
 201     private static final String UPD_WARN = "Update operation not supported; Config store updated;"
 202             + " restore with another Update if needed.";
 203     private static long bgp_as_num = 0;
 204     private static List<Neighbors> nbrList = new ArrayList<>();
 205     private int bgpKaTime = 0;
 206     private int bgpHoldTime = 0;
 207     private int bgpGrRestartTime = 0;
 208
 209     private static final Class<?>[] REACTORS = {
 210         ConfigServerReactor.class, AsIdReactor.class,
 211         GracefulRestartReactor.class, LoggingReactor.class,
 212         NeighborsReactor.class, UpdateSourceReactor.class,
 213         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 214         NetworksReactor.class, VrfsReactor.class, BgpReactor.class,
 215         MultipathReactor.class, VrfMaxpathReactor.class, BfdConfigReactor.class
 216     };
 217
 218     private IBgpManager bgpManager;
 219     private final DataBroker dataBroker;
 220     private final FibDSWriter fibDSWriter;
 221     private final IVpnLinkService vpnLinkService;
 222     private final BundleContext bundleContext;
 223     private final BgpUtil bgpUtil;
 224     private volatile Bgp config;
 225     private final BgpRouter bgpRouter;
 226     private final BgpSyncHandle bgpSyncHandle = new BgpSyncHandle();
 227     private volatile BgpThriftService bgpThriftService = null;
 228     private final int delayEorSeconds;
 229
 230     private final CountDownLatch initer = new CountDownLatch(1);
 231
 232     private final String hostStartup;
 233     private final String portStartup;
 234
 235     private final AtomicReference<BgpCounters> bgpCountersReference = new AtomicReference<>();
 236     private ScheduledFuture<?> bgpCountersTask;
 237
 238     private final AtomicReference<BgpAlarms> bgpAlarmsReference = new AtomicReference<>();
 239     private ScheduledFuture<?> bgpAlarmsTask;
 240
 241     private Future<?> lastReplayJobFt;
 242     private ScheduledFuture<?> routeCleanupFuture;
 243
 244     private long staleStartTime;
 245     private long staleEndTime;
 246     private long cfgReplayStartTime;
 247     private long cfgReplayEndTime;
 248     private long staleCleanupTime;
 249     private int totalStaledCount;
 250     private int totalCleared;
 251     private int totalExternalRoutes;
 252     private int totalExternalMacRoutes;
 253
 254     private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(
 255             new ThreadFactoryBuilder().setNameFormat("bgp-config-%d").setDaemon(true).build());
 256
 257     /**
 258      * this map store the new address families to send to quagga. When it is sended you must clear it.
 259      * The keys String are rd (route distinguisher).
 260      * */
 261     private final ConcurrentHashMap<String, List<AddressFamiliesVrf>> mapNewAdFamily = new ConcurrentHashMap<>();
 262
 263     // map<rd, map<prefix/len:nexthop, label>>
 264     private final Map<String, Map<String, Uint32>> staledFibEntriesMap = new ConcurrentHashMap<>();
 265
 266     // map<rd, map<tep-ip, map<mac, l2vni>>>
 267     private final Map<String, Map<String, Map<String, Uint32>>> rt2TepMap = new ConcurrentHashMap<>();
 268
 269     //map<rd+prefix/plen, list (nexthop)>
 270     private final Map<String,List> fibMap = new HashMap<>();
 271
 272     private final List<AutoCloseable> listeners = new ArrayList<>();
 273
 274     private final EntityOwnershipUtils entityOwnershipUtils;
 275     private final EntityOwnershipCandidateRegistration candidateRegistration;
 276     private final EntityOwnershipListenerRegistration entityListenerRegistration;
 277     private final MetricProvider metricProvider;
 278     private final TransactionHistory bgpUpdatesHistory;
 279
 280     private volatile AtomicBoolean eorSupressedDuetoUpgradeFlag = new AtomicBoolean(false);
 281
 282     @Inject
 283     public BgpConfigurationManager(final DataBroker dataBroker,
 284             final EntityOwnershipService entityOwnershipService,
 285             final FibDSWriter fibDSWriter,
 286             final IVpnLinkService vpnLinkSrvce,
 287             final BundleContext bundleContext,
 288             final BgpUtil bgpUtil,
 289             final MetricProvider metricProvider) {
 290         this.dataBroker = dataBroker;
 291         this.fibDSWriter = fibDSWriter;
 292         this.vpnLinkService = vpnLinkSrvce;
 293         this.bundleContext = bundleContext;
 294         this.bgpUtil = bgpUtil;
 295         this.metricProvider = metricProvider;
 296         hostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 297         portStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 298         bgpKaTime =
 299                 Integer.parseInt(getProperty(BGP_KA_TIMER_PROPERTY,
 300                         Integer.toString(DEF_BGP_KA_TIME)));
 301         bgpHoldTime =
 302                 Integer.parseInt(getProperty(BGP_HOLD_TIMER_PROPERTY,
 303                         Integer.toString(DEF_BGP_HOLD_TIME)));
 304         bgpGrRestartTime =
 305                 Integer.parseInt(getProperty(BGP_GR_RESTART_TIMER_PROPERTY,
 306                         Integer.toString(DEF_BGP_GR_TIME)));
 307         LOG.info("ConfigServer at {}:{}", hostStartup, portStartup);
 308         VtyshCli.setHostAddr(hostStartup);
 309         ClearBgpCli.setHostAddr(hostStartup);
 310         bgpUpdatesHistory = new TransactionHistory(HISTORY_LIMIT, HISTORY_THRESHOLD);
 311         bgpRouter = BgpRouter.newInstance(this::getConfig, this::isBGPEntityOwner, bgpUpdatesHistory);
 312         delayEorSeconds = Integer.parseInt(getProperty(BGP_EOR_DELAY, DEF_BGP_EOR_DELAY));
 313
 314         entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 315
 316         candidateRegistration = registerEntityCandidate(entityOwnershipService);
 317         entityListenerRegistration = registerEntityListener(entityOwnershipService);
 318
 319         LOG.info("BGP Configuration manager initialized");
 320         initer.countDown();
 321
 322         GlobalEventExecutor.INSTANCE.execute(() -> {
 323             ServiceTracker<IBgpManager, ?> tracker = null;
 324             try {
 325                 tracker = new ServiceTracker<>(bundleContext, IBgpManager.class, null);
 326                 tracker.open();
 327                 bgpManager = (IBgpManager) tracker.waitForService(TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES));
 328                 Preconditions.checkState(bgpManager != null, "IBgpManager service not found");
 329             } catch (InterruptedException e) {
 330                 throw new IllegalStateException("Error retrieving IBgpManager service", e);
 331             } finally {
 332                 if (tracker != null) {
 333                     tracker.close();
 334                 }
 335             }
 336
 337             String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
 338             if (InetAddresses.isInetAddress(getBgpSdncMipIp())) {
 339                 InetSocketAddress bgpThriftServerSocketAddr = new InetSocketAddress(getBgpSdncMipIp(),
 340                         Integer.parseInt(updatePort));
 341                 bgpThriftService = new BgpThriftService(bgpThriftServerSocketAddr, bgpManager, this);
 342                 if (isBGPEntityOwner()) {
 343                     //I am EoS owner of BGP, opening bgp thrift UPDATE-SERVER port.
 344                     LOG.info("BGP Configuration manager initialized: UPDATE-SERVER started");
 345                     bgpThriftService.start();
 346                 }
 347                 LOG.info("UPDATE server started :ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 348             } else {
 349                 LOG.error("Failed to init UPDATE server invalid ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 350             }
 351         });
 352         registerCallbacks();
 353         LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 354     }
 355
 356     public String getBgpSdncMipIp() {
 357         return getProperty(BGP_SDNC_MIP, DEF_BGP_SDNC_MIP);
 358     }
 359
 360     public long getStaleCleanupTime() {
 361         return staleCleanupTime;
 362     }
 363
 364     public void setStaleCleanupTime(long staleCleanupTime) {
 365         this.staleCleanupTime = staleCleanupTime;
 366     }
 367
 368     public long getCfgReplayEndTime() {
 369         return cfgReplayEndTime;
 370     }
 371
 372     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 373         this.cfgReplayEndTime = cfgReplayEndTime;
 374     }
 375
 376     public TransactionHistory getBgpUpdatesHistory() {
 377         return bgpUpdatesHistory;
 378     }
 379
 380     public long getCfgReplayStartTime() {
 381         return cfgReplayStartTime;
 382     }
 383
 384     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 385         this.cfgReplayStartTime = cfgReplayStartTime;
 386     }
 387
 388     public long getStaleEndTime() {
 389         return staleEndTime;
 390     }
 391
 392     public void setStaleEndTime(long staleEndTime) {
 393         this.staleEndTime = staleEndTime;
 394     }
 395
 396     public long getStaleStartTime() {
 397         return staleStartTime;
 398     }
 399
 400     public void setStaleStartTime(long staleStartTime) {
 401         this.staleStartTime = staleStartTime;
 402     }
 403
 404     private Object createListener(Class<?> cls) {
 405         try {
 406             Constructor<?> ctor = cls.getConstructor(BgpConfigurationManager.class);
 407             return ctor.newInstance(this);
 408         } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException
 409                 e) {
 410             LOG.error("Failed to create listener object", e);
 411             return null;
 412         }
 413     }
 414
 415     private void registerCallbacks() {
 416         for (Class<?> reactor : REACTORS) {
 417             Object obj = createListener(reactor);
 418             if (obj != null) {
 419                 AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 420                 dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 421                 listeners.add(dcl);
 422             }
 423         }
 424     }
 425
 426     @SuppressWarnings("checkstyle:IllegalCatch")
 427     @PreDestroy
 428     public void close() {
 429         executor.shutdown();
 430
 431         if (bgpThriftService != null) {
 432             bgpThriftService.stop();
 433             bgpThriftService = null;
 434         }
 435
 436         if (isBgpConnected()) {
 437             //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 438             bgpRouter.disconnect();
 439         }
 440
 441         if (candidateRegistration != null) {
 442             candidateRegistration.close();
 443         }
 444
 445         entityListenerRegistration.close();
 446
 447         listeners.forEach(l -> {
 448             try {
 449                 l.close();
 450             } catch (Exception e) {
 451                 LOG.warn("Error closing {}", l ,e);
 452             }
 453         });
 454
 455         LOG.info("{} close", getClass().getSimpleName());
 456     }
 457
 458     private String getProperty(String var, String def) {
 459         String property = bundleContext.getProperty(var);
 460         return property == null ? def : property;
 461     }
 462
 463     private EntityOwnershipCandidateRegistration registerEntityCandidate(
 464             final EntityOwnershipService entityOwnershipService) {
 465         try {
 466             return entityOwnershipService.registerCandidate(
 467                     new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME));
 468         } catch (CandidateAlreadyRegisteredException e) {
 469             LOG.error("failed to register bgp entity", e);
 470             return null;
 471         }
 472     }
 473
 474     private EntityOwnershipListenerRegistration registerEntityListener(
 475             final EntityOwnershipService entityOwnershipService) {
 476         return entityOwnershipService.registerListener(BGP_ENTITY_TYPE_FOR_OWNERSHIP, ownershipChange -> {
 477             LOG.info("entity owner change event fired: {}", ownershipChange);
 478
 479             if (ownershipChange.getState() == EntityOwnershipChangeState.LOCAL_OWNERSHIP_GRANTED) {
 480                 LOG.info("This PL is the Owner");
 481                 if (bgpThriftService != null) {
 482                     //opening UPDATE-SERVER port.
 483                     bgpThriftService.start();
 484                 } else {
 485                     LOG.error("I am the owner of BGP entity, but bgpThriftService is not initialized yet");
 486                 }
 487                 bgpRestarted();
 488             } else {
 489                 LOG.info("Not owner: hasOwner: {}, isOwner: {}", ownershipChange.getState().hasOwner(),
 490                         ownershipChange.getState().isOwner());
 491                 if (bgpThriftService != null && bgpThriftService.isBgpThriftServiceStarted()) {
 492                     //close the bgp Thrift Update-SERVER port opened on non-Entity Owner
 493                     bgpThriftService.stop();
 494                 }
 495                 if (isBgpConnected()) {
 496                     //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 497                     bgpRouter.disconnect();
 498                 }
 499                 stopBgpCountersTask();
 500                 stopBgpAlarmsTask();
 501             }
 502         });
 503     }
 504
 505     public boolean isBGPEntityOwner() {
 506         if (entityOwnershipUtils == null) {
 507             LOG.error("entityOwnershipUtils is NULL when listener callbacks fired");
 508             return false;
 509         }
 510         return entityOwnershipUtils.isEntityOwner(new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME), 0, 1);
 511     }
 512
 513     public Bgp get() {
 514         config = getConfig();
 515         return config;
 516     }
 517
 518     public class ConfigServerReactor
 519             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 520             implements ClusteredDataTreeChangeListener<ConfigServer> {
 521         private static final String YANG_OBJ = "config-server ";
 522
 523         public ConfigServerReactor() {
 524             super(ConfigServer.class, ConfigServerReactor.class);
 525         }
 526
 527         @Override
 528         protected void add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 529             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 530             if (!isBGPEntityOwner()) {
 531                 return;
 532             }
 533
 534             try {
 535                 initer.await();
 536             } catch (InterruptedException e) {
 537                 // Ignored
 538             }
 539             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 540             bgpRouter.configServerUpdated();
 541             synchronized (BgpConfigurationManager.this) {
 542                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 543                         val.getPort().intValue());
 544                 if (!res) {
 545                     LOG.error(YANG_OBJ + "Add failed; " + ADD_WARN);
 546                 }
 547             }
 548             VtyshCli.setHostAddr(val.getHost().getValue());
 549             ClearBgpCli.setHostAddr(val.getHost().getValue());
 550         }
 551
 552         @Override
 553         protected ConfigServerReactor getDataTreeChangeListener() {
 554             return ConfigServerReactor.this;
 555         }
 556
 557         @Override
 558         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 559             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 560         }
 561
 562         @Override
 563         protected void remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 564             LOG.trace("received bgp disconnect");
 565             if (!isBGPEntityOwner()) {
 566                 return;
 567             }
 568
 569             bgpRouter.configServerUpdated();
 570
 571             synchronized (BgpConfigurationManager.this) {
 572                 if (bgp_as_num != 0) {
 573                     try {
 574                         bgpRouter.stopBgp(bgp_as_num);
 575                         stopBgpCountersTask();
 576                         stopBgpAlarmsTask();
 577                     } catch (TException | BgpRouterException e) {
 578                         LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 579                     }
 580                 } else {
 581                     LOG.debug("bgp as-id is null while removing config-server");
 582                 }
 583                 bgpRouter.disconnect();
 584             }
 585         }
 586
 587         @Override
 588         protected void update(InstanceIdentifier<ConfigServer> iid,
 589                 ConfigServer oldval, ConfigServer newval) {
 590             LOG.trace("received bgp Connection update");
 591             if (!isBGPEntityOwner()) {
 592                 return;
 593             }
 594             LOG.error(YANG_OBJ + UPD_WARN);
 595         }
 596     }
 597
 598     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
 599             justification = "https://github.com/spotbugs/spotbugs/issues/811")
 600     private BgpRouter getClient(String yangObj) {
 601         if (bgpRouter == null || !bgpRouter.isBgpConnected()) {
 602             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 603             return null;
 604         }
 605         return bgpRouter;
 606     }
 607
 608     public class AsIdReactor
 609             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 610             implements ClusteredDataTreeChangeListener<AsId> {
 611
 612         private static final String YANG_OBJ = "as-id ";
 613
 614         public AsIdReactor() {
 615             super(AsId.class, AsIdReactor.class);
 616         }
 617
 618         @Override
 619         protected void add(InstanceIdentifier<AsId> iid, AsId val) {
 620             LOG.error("received bgp add asid {}", val);
 621             if (!isBGPEntityOwner()) {
 622                 return;
 623             }
 624             LOG.debug("received add router config asNum {}", val.getLocalAs());
 625             bgp_as_num = val.getLocalAs().longValue();
 626             synchronized (BgpConfigurationManager.this) {
 627                 BgpRouter br = getClient(YANG_OBJ);
 628                 if (br == null) {
 629                     LOG.debug("{} Unable to process add for asNum {}; {} {}", YANG_OBJ, val.getLocalAs(),
 630                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 631                     return;
 632                 }
 633                 bgpRestarted();
 634
 635                 startBgpCountersTask();
 636                 startBgpAlarmsTask();
 637             }
 638         }
 639
 640         @Override
 641         protected AsIdReactor getDataTreeChangeListener() {
 642             return AsIdReactor.this;
 643         }
 644
 645         @Override
 646         protected InstanceIdentifier<AsId> getWildCardPath() {
 647             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 648         }
 649
 650         @Override
 651         protected void remove(InstanceIdentifier<AsId> iid, AsId val) {
 652             LOG.error("received delete router config asNum {}", val.getLocalAs());
 653             if (!isBGPEntityOwner()) {
 654                 return;
 655             }
 656             synchronized (BgpConfigurationManager.this) {
 657                 long asNum = val.getLocalAs().toJava();
 658                 BgpRouter br = getClient(YANG_OBJ);
 659                 bgp_as_num = 0;
 660                 if (br == null) {
 661                     LOG.debug("{} Unable to process remove for asNum {}; {} {}", YANG_OBJ, asNum,
 662                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 663                     return;
 664                 }
 665                 try {
 666                     br.stopBgp(asNum);
 667                 } catch (TException | BgpRouterException e) {
 668                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 669                 }
 670
 671                 stopBgpCountersTask();
 672                 stopBgpAlarmsTask();
 673
 674                 Bgp conf = getConfig();
 675                 if (conf == null) {
 676                     LOG.error("Config Null while removing the as-id");
 677                     return;
 678                 }
 679                 LOG.debug("Removing external routes from FIB");
 680                 deleteExternalFibRoutes();
 681                 List<Neighbors> nbrs = conf.getNeighborsContainer() == null ? null
 682                         : conf.getNeighborsContainer().getNeighbors();
 683                 if (nbrs != null && nbrs.size() > 0) {
 684                     LOG.error("Tring to remove the as-id when neighbor config is already present");
 685                     for (Neighbors nbr : nbrs) {
 686                         LOG.debug("Removing Neighbor {} from Data store", nbr.getAddress().getValue());
 687                         delNeighbor(nbr.getAddress().getValue());
 688                     }
 689                 }
 690             }
 691         }
 692
 693         @Override
 694         protected void update(InstanceIdentifier<AsId> iid,
 695                 AsId oldval, AsId newval) {
 696             if (!isBGPEntityOwner()) {
 697                 return;
 698             }
 699             LOG.error(YANG_OBJ + UPD_WARN);
 700         }
 701     }
 702
 703     public class GracefulRestartReactor
 704             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 705             implements ClusteredDataTreeChangeListener<GracefulRestart> {
 706
 707         private static final String YANG_OBJ = "graceful-restart ";
 708
 709         public GracefulRestartReactor() {
 710             super(GracefulRestart.class, GracefulRestartReactor.class);
 711         }
 712
 713         @Override
 714         protected void add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 715             if (!isBGPEntityOwner()) {
 716                 return;
 717             }
 718             synchronized (BgpConfigurationManager.this) {
 719                 int stalePathTime = val.getStalepathTime().intValue();
 720                 BgpRouter br = getClient(YANG_OBJ);
 721                 if (br == null) {
 722                     LOG.error("{} Unable to add stale-path time {}; {} {}", YANG_OBJ, stalePathTime,
 723                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 724                     return;
 725                 }
 726                 try {
 727                     br.addGracefulRestart(stalePathTime);
 728                 } catch (TException | BgpRouterException e) {
 729                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 730                 }
 731             }
 732         }
 733
 734         @Override
 735         protected GracefulRestartReactor getDataTreeChangeListener() {
 736             return GracefulRestartReactor.this;
 737         }
 738
 739         @Override
 740         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 741             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 742         }
 743
 744         @Override
 745         protected void remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 746             if (!isBGPEntityOwner()) {
 747                 return;
 748             }
 749             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 750             synchronized (BgpConfigurationManager.this) {
 751                 BgpRouter br = getClient(YANG_OBJ);
 752                 if (br == null) {
 753                     LOG.error("{} Unable to delete stale-path time; {} {}", YANG_OBJ,
 754                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 755                     return;
 756                 }
 757                 try {
 758                     br.delGracefulRestart();
 759                 } catch (TException | BgpRouterException e) {
 760                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 761                 }
 762             }
 763         }
 764
 765         @Override
 766         protected void update(InstanceIdentifier<GracefulRestart> iid,
 767                 GracefulRestart oldval, GracefulRestart newval) {
 768             if (!isBGPEntityOwner()) {
 769                 return;
 770             }
 771             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 772             synchronized (BgpConfigurationManager.this) {
 773                 int stalePathTime = newval.getStalepathTime().intValue();
 774                 BgpRouter br = getClient(YANG_OBJ);
 775                 if (br == null) {
 776                     LOG.error("{} Unable to update stale-path time to {}; {} {}", YANG_OBJ, stalePathTime,
 777                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 778                     return;
 779                 }
 780                 try {
 781                     br.addGracefulRestart(stalePathTime);
 782                 } catch (TException | BgpRouterException e) {
 783                     LOG.error("{} update received exception; {}", YANG_OBJ, ADD_WARN, e);
 784                 }
 785             }
 786         }
 787     }
 788
 789     public class LoggingReactor
 790             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 791             implements ClusteredDataTreeChangeListener<Logging> {
 792
 793         private static final String YANG_OBJ = "logging ";
 794
 795         public LoggingReactor() {
 796             super(Logging.class, LoggingReactor.class);
 797         }
 798
 799         @Override
 800         protected void add(InstanceIdentifier<Logging> iid, Logging val) {
 801             if (!isBGPEntityOwner()) {
 802                 return;
 803             }
 804             synchronized (BgpConfigurationManager.this) {
 805                 BgpRouter br = getClient(YANG_OBJ);
 806                 if (br == null) {
 807                     LOG.error("{} Unable to add logging for qbgp; {} {}", YANG_OBJ,
 808                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 809                     return;
 810                 }
 811                 try {
 812                     br.setLogging(val.getFile(), val.getLevel());
 813                 } catch (TException | BgpRouterException e) {
 814                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 815                 }
 816             }
 817         }
 818
 819         @Override
 820         protected LoggingReactor getDataTreeChangeListener() {
 821             return LoggingReactor.this;
 822         }
 823
 824         @Override
 825         protected InstanceIdentifier<Logging> getWildCardPath() {
 826             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 827         }
 828
 829         @Override
 830         protected void remove(InstanceIdentifier<Logging> iid, Logging val) {
 831             if (!isBGPEntityOwner()) {
 832                 return;
 833             }
 834             LOG.debug("received remove Logging config val {}", val.getLevel());
 835             synchronized (BgpConfigurationManager.this) {
 836                 BgpRouter br = getClient(YANG_OBJ);
 837                 if (br == null) {
 838                     LOG.error("{} Unable to remove logging for qbgp; {} {}", YANG_OBJ,
 839                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 840                     return;
 841                 }
 842                 try {
 843                     br.setLogging(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
 844                 } catch (TException | BgpRouterException e) {
 845                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 846                 }
 847             }
 848         }
 849
 850         @Override
 851         protected void update(InstanceIdentifier<Logging> iid,
 852                 Logging oldval, Logging newval) {
 853             if (!isBGPEntityOwner()) {
 854                 return;
 855             }
 856             synchronized (BgpConfigurationManager.this) {
 857                 BgpRouter br = getClient(YANG_OBJ);
 858                 if (br == null) {
 859                     LOG.error("{} Unable to update logging for qbgp; {} {}", YANG_OBJ,
 860                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 861                     return;
 862                 }
 863                 try {
 864                     br.setLogging(newval.getFile(), newval.getLevel());
 865                 } catch (TException | BgpRouterException e) {
 866                     LOG.error("{} newval received exception; {}", YANG_OBJ, ADD_WARN, e);
 867                 }
 868             }
 869         }
 870     }
 871
 872     public class NeighborsReactor
 873             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 874             implements ClusteredDataTreeChangeListener<Neighbors> {
 875
 876         private static final String YANG_OBJ = "neighbors ";
 877
 878         public NeighborsReactor() {
 879             super(Neighbors.class, NeighborsReactor.class);
 880         }
 881
 882         @Override
 883         protected void add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 884             if (nbrList != null && !nbrList.contains(val)) {
 885                 LOG.trace("Adding nbr {} to nbrlist", val.getAddress().getValue());
 886                 nbrList.add(val);
 887             }
 888             if (!isBGPEntityOwner()) {
 889                 return;
 890             }
 891             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 892             synchronized (BgpConfigurationManager.this) {
 893                 String peerIp = val.getAddress().getValue();
 894                 String sourceIp = (val.getUpdateSource() == null) ? null :
 895                                     val.getUpdateSource().getSourceIp().getValue();
 896                 int nhops = (val.getEbgpMultihop() == null) ? 0 :
 897                                     val.getEbgpMultihop().getNhops().intValue();
 898                 List<AddressFamilies> afs = val.getAddressFamilies();
 899                 long as = val.getRemoteAs().toJava();
 900                 final String md5Secret = extractMd5Secret(val);
 901                 BgpRouter br = getClient(YANG_OBJ);
 902                 if (br == null) {
 903                     LOG.debug("{} Unable to process add for peer {} as {}; {} {}", YANG_OBJ, peerIp, as,
 904                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 905                     return;
 906                 }
 907                 try {
 908                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 909                     br.addNeighbor(peerIp, as, md5Secret);
 910                     if (nhops != 0) {
 911                         br.addEbgpMultihop(peerIp, nhops);
 912                     }
 913                     if (sourceIp != null) {
 914                         br.addUpdateSource(peerIp, sourceIp);
 915                     }
 916                     if (afs != null) {
 917                         for (AddressFamilies af : afs) {
 918                             af_afi afi = af_afi.findByValue(af.getAfi().intValue());
 919                             af_safi safi = af_safi.findByValue(af.getSafi().intValue());
 920                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
 921                         }
 922                     }
 923
 924                 } catch (TException | BgpRouterException e) {
 925                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 926                 }
 927             }
 928         }
 929
 930         @Override
 931         protected NeighborsReactor getDataTreeChangeListener() {
 932             return NeighborsReactor.this;
 933         }
 934
 935         @Override
 936         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 937             return InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class);
 938         }
 939
 940         @Override
 941         protected void remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 942             if (nbrList != null && nbrList.contains(val)) {
 943                 LOG.trace("Removing nbr {} from nbr list", val.getAddress().getValue());
 944                 nbrList.remove(val);
 945             }
 946             if (!isBGPEntityOwner()) {
 947                 return;
 948             }
 949             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 950             synchronized (BgpConfigurationManager.this) {
 951                 String peerIp = val.getAddress().getValue();
 952                 BgpRouter br = getClient(YANG_OBJ);
 953                 if (br == null) {
 954                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 955                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 956                     return;
 957                 }
 958                 try {
 959                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 960                     br.delNeighbor(peerIp);
 961                 } catch (TException | BgpRouterException e) {
 962                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 963                 }
 964
 965                 final BgpAlarms bgpAlarms = getBgpAlarms();
 966                 if (bgpAlarms != null) {
 967                     bgpAlarms.clearBgpNbrDownAlarm(peerIp);
 968                 }
 969
 970                 if (bgpUtil.isBfdEnabled()) {
 971                     final BgpCounters bgpCounters = getBgpCounters();
 972                     if (bgpCounters != null) {
 973                         bgpCounters.clearBfdNbrCounters(peerIp);
 974                     }
 975                 }
 976             }
 977         }
 978
 979         @Override
 980         protected void update(InstanceIdentifier<Neighbors> iid,
 981                 Neighbors oldval, Neighbors newval) {
 982             if (!isBGPEntityOwner()) {
 983                 return;
 984             }
 985             //purposefully nothing to do.
 986         }
 987     }
 988
 989     public class EbgpMultihopReactor
 990             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 991             implements ClusteredDataTreeChangeListener<EbgpMultihop> {
 992
 993         private static final String YANG_OBJ = "ebgp-multihop ";
 994
 995         public EbgpMultihopReactor() {
 996             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 997         }
 998
 999         @Override
1000         protected void add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
1001             if (!isBGPEntityOwner()) {
1002                 return;
1003             }
1004             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
1005             synchronized (BgpConfigurationManager.this) {
1006                 String peerIp = val.getPeerIp().getValue();
1007                 BgpRouter br = getClient(YANG_OBJ);
1008                 if (br == null) {
1009                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1010                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1011                     return;
1012                 }
1013                 try {
1014                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
1015                 } catch (TException | BgpRouterException e) {
1016                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1017                 }
1018             }
1019         }
1020
1021         @Override
1022         protected EbgpMultihopReactor getDataTreeChangeListener() {
1023             return EbgpMultihopReactor.this;
1024         }
1025
1026         @Override
1027         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
1028             return InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
1029                     .child(EbgpMultihop.class);
1030         }
1031
1032         @Override
1033         protected void remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
1034             if (!isBGPEntityOwner()) {
1035                 return;
1036             }
1037             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
1038             synchronized (BgpConfigurationManager.this) {
1039                 String peerIp = val.getPeerIp().getValue();
1040                 BgpRouter br = getClient(YANG_OBJ);
1041                 if (br == null) {
1042                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1043                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1044                     return;
1045                 }
1046                 try {
1047                     br.delEbgpMultihop(peerIp);
1048                 } catch (TException | BgpRouterException e) {
1049                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1050                 }
1051             }
1052         }
1053
1054         @Override
1055         protected void update(InstanceIdentifier<EbgpMultihop> iid,
1056                 EbgpMultihop oldval, EbgpMultihop newval) {
1057             if (!isBGPEntityOwner()) {
1058                 return;
1059             }
1060             LOG.error(YANG_OBJ + UPD_WARN);
1061         }
1062     }
1063
1064     public class UpdateSourceReactor
1065             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
1066             implements ClusteredDataTreeChangeListener<UpdateSource> {
1067
1068         private static final String YANG_OBJ = "update-source ";
1069
1070         public UpdateSourceReactor() {
1071             super(UpdateSource.class, UpdateSourceReactor.class);
1072         }
1073
1074         @Override
1075         protected void add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
1076             if (!isBGPEntityOwner()) {
1077                 return;
1078             }
1079             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
1080             synchronized (BgpConfigurationManager.this) {
1081                 String peerIp = val.getPeerIp().getValue();
1082                 BgpRouter br = getClient(YANG_OBJ);
1083                 if (br == null) {
1084                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1085                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1086                     return;
1087                 }
1088                 try {
1089                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
1090                 } catch (TException | BgpRouterException e) {
1091                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1092                 }
1093             }
1094         }
1095
1096         @Override
1097         protected UpdateSourceReactor getDataTreeChangeListener() {
1098             return UpdateSourceReactor.this;
1099         }
1100
1101         @Override
1102         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
1103             return InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
1104                     .child(UpdateSource.class);
1105         }
1106
1107         @Override
1108         protected void remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
1109             if (!isBGPEntityOwner()) {
1110                 return;
1111             }
1112             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
1113             synchronized (BgpConfigurationManager.this) {
1114                 String peerIp = val.getPeerIp().getValue();
1115                 BgpRouter br = getClient(YANG_OBJ);
1116                 if (br == null) {
1117                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1118                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1119                     return;
1120                 }
1121                 try {
1122                     br.delUpdateSource(peerIp);
1123                 } catch (TException | BgpRouterException e) {
1124                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1125                 }
1126             }
1127         }
1128
1129         @Override
1130         protected void update(InstanceIdentifier<UpdateSource> iid,
1131                 UpdateSource oldval, UpdateSource newval) {
1132             if (!isBGPEntityOwner()) {
1133                 return;
1134             }
1135             LOG.error(YANG_OBJ + UPD_WARN);
1136         }
1137     }
1138
1139     public class AddressFamiliesReactor
1140             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
1141             implements ClusteredDataTreeChangeListener<AddressFamilies> {
1142
1143         private static final String YANG_OBJ = "address-families ";
1144
1145         public AddressFamiliesReactor() {
1146             super(AddressFamilies.class, AddressFamiliesReactor.class);
1147         }
1148
1149         @Override
1150         protected void add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1151             if (!isBGPEntityOwner()) {
1152                 return;
1153             }
1154             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
1155             synchronized (BgpConfigurationManager.this) {
1156                 String peerIp = val.getPeerIp().getValue();
1157                 BgpRouter br = getClient(YANG_OBJ);
1158                 if (br == null) {
1159                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1160                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1161                     return;
1162                 }
1163                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1164                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1165                 try {
1166                     br.addAddressFamily(peerIp, afi, safi);
1167                 } catch (TException | BgpRouterException e) {
1168                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1169                 }
1170             }
1171         }
1172
1173         @Override
1174         protected AddressFamiliesReactor getDataTreeChangeListener() {
1175             return AddressFamiliesReactor.this;
1176         }
1177
1178         @Override
1179         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
1180             return InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
1181                     .child(AddressFamilies.class);
1182         }
1183
1184         @Override
1185         protected void remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1186             if (!isBGPEntityOwner()) {
1187                 return;
1188             }
1189             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
1190             synchronized (BgpConfigurationManager.this) {
1191                 String peerIp = val.getPeerIp().getValue();
1192                 BgpRouter br = getClient(YANG_OBJ);
1193                 if (br == null) {
1194                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1195                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1196                     return;
1197                 }
1198                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1199                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1200                 try {
1201                     br.delAddressFamily(peerIp, afi, safi);
1202                 } catch (TException | BgpRouterException e) {
1203                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1204                 }
1205             }
1206         }
1207
1208         @Override
1209         protected void update(InstanceIdentifier<AddressFamilies> iid,
1210                 AddressFamilies oldval, AddressFamilies newval) {
1211             if (!isBGPEntityOwner()) {
1212                 return;
1213             }
1214             LOG.error(YANG_OBJ + UPD_WARN);
1215         }
1216     }
1217
1218     public class NetworksReactor
1219             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
1220             implements ClusteredDataTreeChangeListener<Networks> {
1221
1222         private static final String YANG_OBJ = "networks ";
1223
1224         public NetworksReactor() {
1225             super(Networks.class, NetworksReactor.class);
1226         }
1227
1228         @Override
1229         public NetworksReactor getDataTreeChangeListener() {
1230             return NetworksReactor.this;
1231         }
1232
1233         @Override
1234         protected void add(InstanceIdentifier<Networks> iid, Networks val) {
1235             if (!isBGPEntityOwner()) {
1236                 return;
1237             }
1238             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1239             synchronized (BgpConfigurationManager.this) {
1240                 String rd = val.getRd();
1241                 String pfxlen = val.getPrefixLen();
1242                 String nh = val.getNexthop().getValue();
1243                 BgpRouter br = getClient(YANG_OBJ);
1244                 if (br == null) {
1245                     LOG.debug("{} Unable to process add for rd {} prefix {} nexthop {}; {} {}", YANG_OBJ, rd, pfxlen,
1246                             nh, BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1247                     return;
1248                 }
1249                 Long label = val.getLabel().toJava();
1250                 int lbl = label == null ? qbgpConstants.LBL_NO_LABEL
1251                         : label.intValue();
1252                 int l3vni = val.getL3vni() == null ? qbgpConstants.LBL_NO_LABEL
1253                         : val.getL3vni().intValue();
1254                 int l2vni = val.getL2vni() == null ? qbgpConstants.LBL_NO_LABEL
1255                         : val.getL2vni().intValue();
1256
1257                 BgpControlPlaneType protocolType = val.getBgpControlPlaneType();
1258                 int ethernetTag = val.getEthtag().intValue();
1259                 String esi = val.getEsi();
1260                 String macaddress = val.getMacaddress();
1261                 EncapType encapType = val.getEncapType();
1262                 String routerMac = val.getRoutermac();
1263
1264                 try {
1265                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni, BgpUtil.convertToThriftProtocolType(protocolType),
1266                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1267                 } catch (TException | BgpRouterException e) {
1268                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1269                 }
1270             }
1271         }
1272
1273         @Override
1274         protected InstanceIdentifier<Networks> getWildCardPath() {
1275             return InstanceIdentifier.create(Bgp.class).child(NetworksContainer.class).child(Networks.class);
1276         }
1277
1278         @Override
1279         protected void remove(InstanceIdentifier<Networks> iid, Networks val) {
1280             if (!isBGPEntityOwner()) {
1281                 return;
1282             }
1283             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1284             synchronized (BgpConfigurationManager.this) {
1285                 String rd = val.getRd();
1286                 String pfxlen = val.getPrefixLen();
1287                 BgpRouter br = getClient(YANG_OBJ);
1288                 if (br == null) {
1289                     LOG.debug("{} Unable to process remove for rd {} prefix {}; {} {}", YANG_OBJ, rd, pfxlen,
1290                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1291                     return;
1292                 }
1293                 Long label = val.getLabel().toJava();
1294                 int lbl = label == null ? 0 : label.intValue();
1295                 if (rd == null && lbl > 0) {
1296                     //LU prefix is being deleted.
1297                     rd = Integer.toString(lbl);
1298                 }
1299                 try {
1300                     br.delPrefix(rd, pfxlen);
1301                 } catch (TException | BgpRouterException e) {
1302                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1303                 }
1304             }
1305         }
1306
1307         /**get the value AFI from a prefix as "x.x.x.x/x".
1308          *
1309          * @param pfxlen the prefix to get an afi
1310          * @return the afi value as you are need
1311          */
1312         public  int  testValueAFI(String pfxlen) {
1313             int afiNew = af_afi.AFI_IP.getValue();
1314             try {
1315                 String ipOnly = pfxlen.substring(0, pfxlen.lastIndexOf("/"));
1316                 java.net.Inet6Address.getByName(ipOnly);
1317                 afiNew = af_afi.AFI_IPV6.getValue();
1318             } catch (java.net.UnknownHostException e) {
1319                 //ce n'est pas de l'ipv6
1320             }
1321             return afiNew;
1322         }
1323
1324
1325         @Override
1326         protected void update(final InstanceIdentifier<Networks> iid,
1327                 final Networks oldval, final Networks newval) {
1328             if (!isBGPEntityOwner()) {
1329                 return;
1330             }
1331             if (oldval.equals(newval)) {
1332                 //Update: OLD and New values are same, no need to trigger remove/add.
1333                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1334                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1335                 return;
1336             }
1337             LOG.debug("update networks old val RD: {}, Prefix: {}, Label: {}, NH: {} "
1338                             + "new val RD: {}, Prefix: {}, Label: {}, NH: {}",
1339                     oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop(),
1340                     newval.getRd(), newval.getPrefixLen(), newval.getLabel(), newval.getNexthop());
1341             remove(iid, oldval);
1342             add(iid, newval);
1343         }
1344     }
1345
1346     static Timer timer = new Timer();
1347
1348     public class VrfsReactor
1349             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1350             implements ClusteredDataTreeChangeListener<Vrfs> {
1351
1352         private static final String YANG_OBJ = "vrfs ";
1353
1354         public VrfsReactor() {
1355             super(Vrfs.class, VrfsReactor.class);
1356         }
1357
1358         @Override
1359         protected void add(InstanceIdentifier<Vrfs> iid, Vrfs vrfs) {
1360             if (!isBGPEntityOwner()) {
1361                 return;
1362             }
1363             LOG.debug("received add Vrfs config value {}", vrfs.getRd());
1364             synchronized (BgpConfigurationManager.this) {
1365                 String rd = vrfs.getRd();
1366                 BgpRouter br = getClient(YANG_OBJ);
1367                 if (br == null) {
1368                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1369                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1370                     return;
1371                 }
1372                 try {
1373                     List<AddressFamiliesVrf> vrfAddrFamilyList = vrfs.getAddressFamiliesVrf();
1374                     for (AddressFamiliesVrf vrfAddrFamily : vrfAddrFamilyList) {
1375                         /*add to br the new vrfs arguments*/
1376                         br.addVrf(BgpUtil.getLayerType(vrfAddrFamily), rd, vrfs.getImportRts(),
1377                                 vrfs.getExportRts(), vrfAddrFamily.getAfi().toJava(), vrfAddrFamily.getSafi().toJava());
1378                     }
1379                     /*add to br the vrfs contained in mapNewAdFamily*/
1380                     List<AddressFamiliesVrf> vrfAddrFamilyListFromMap = mapNewAdFamily.get(rd);
1381                     if (vrfAddrFamilyListFromMap == null) {
1382                         return;
1383                     }
1384
1385                     for (AddressFamiliesVrf adf : vrfAddrFamilyListFromMap) {
1386                         if (vrfAddrFamilyList.contains(adf)) {
1387                             mapNewAdFamily.remove(rd);
1388                         } else  if (adf != null) {
1389
1390                             br.addVrf(BgpUtil.getLayerType(adf), rd, vrfs.getImportRts(),
1391                                     vrfs.getExportRts(), adf.getAfi().toJava(), adf.getSafi().toJava());
1392                             // remove AddressFamiliesVrf which was already added to BGP
1393                             vrfAddrFamilyListFromMap.remove(adf);
1394                             if (vrfAddrFamilyListFromMap.isEmpty()) {
1395                                 // remove Vrf entry from temp mapNewAdFamily if all its AddressFamiliesVrf was
1396                                 // added to BGP
1397                                 mapNewAdFamily.remove(rd);
1398                             }
1399                         }
1400                     }
1401                 } catch (TException | BgpRouterException e) {
1402                     LOG.error("{} get {}, Add received exception", YANG_OBJ, ADD_WARN, e);
1403                 }
1404             }
1405         }
1406
1407         @Override
1408         protected VrfsReactor getDataTreeChangeListener() {
1409             return VrfsReactor.this;
1410         }
1411
1412         @Override
1413         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1414             return InstanceIdentifier.create(Bgp.class).child(VrfsContainer.class).child(Vrfs.class);
1415         }
1416
1417         @Override
1418         protected void remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1419             if (!isBGPEntityOwner()) {
1420                 return;
1421             }
1422             LOG.debug("received remove Vrfs config val {}", val.getRd());
1423             synchronized (BgpConfigurationManager.this) {
1424                 String rd = val.getRd();
1425                 BgpRouter br = getClient(YANG_OBJ);
1426                 if (br == null) {
1427                     LOG.debug("{} Unable to process remove for rd {}; {} {}", YANG_OBJ, rd,
1428                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1429                     return;
1430                 }
1431                 try {
1432                     List<AddressFamiliesVrf> adf = mapNewAdFamily.get(rd);
1433                     adf = adf != null ? adf : new ArrayList<>();
1434                     for (AddressFamiliesVrf s : val.getAddressFamiliesVrf()) {
1435                         br.delVrf(rd, s.getAfi().toJava(), s.getSafi().toJava());
1436                         adf.remove(s);// remove in the map the vrf in waiting for advertise quagga
1437                     }
1438                     if (adf.isEmpty()) {
1439                         mapNewAdFamily.remove(rd);
1440                     }
1441                 } catch (TException | BgpRouterException e) {
1442                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1443                 }
1444             }
1445         }
1446
1447         @Override
1448         protected void update(InstanceIdentifier<Vrfs> iid,
1449                 Vrfs oldval, Vrfs newval) {
1450             if (oldval != null && newval != null) {
1451                 LOG.debug("received update Vrfs config val {}, VRFS: Update getting triggered for VRFS rd {}",
1452                         newval.getRd(), oldval.getRd());
1453             } else {
1454                 LOG.debug("received update Vrfs config val {}, from old vrf {}",
1455                         newval, oldval);
1456             }
1457             if (!isBGPEntityOwner()) {
1458                 return;
1459             }
1460
1461             List<AddressFamiliesVrf> adFamilyVrfToDel = new ArrayList<>();
1462             List<AddressFamiliesVrf> adFamilyVrfToAdd = new ArrayList<>();
1463             List<AddressFamiliesVrf> oldlistAdFamilies = new ArrayList<>();
1464             List<AddressFamiliesVrf> newlistAdFamilies = new ArrayList<>();
1465             if (oldval != null) {
1466                 oldlistAdFamilies = oldval.getAddressFamiliesVrf() == null
1467                         ? new ArrayList<>() : oldval.getAddressFamiliesVrf();
1468             }
1469             if (newval != null) {
1470                 newlistAdFamilies = newval.getAddressFamiliesVrf() == null
1471                         ? new ArrayList<>() : newval.getAddressFamiliesVrf();
1472             }
1473             /*find old AddressFamily to remove from new configuration*/
1474             for (AddressFamiliesVrf adVrf : oldlistAdFamilies) {
1475                 if (!newlistAdFamilies.contains(adVrf)) {
1476                     adFamilyVrfToDel.add(adVrf);
1477                 }
1478             }
1479             /*find new AddressFamily to add to unexisting configuration*/
1480             for (AddressFamiliesVrf adVrf : newlistAdFamilies) {
1481                 if (!oldlistAdFamilies.contains(adVrf)) {
1482                     adFamilyVrfToAdd.add(adVrf);
1483                 }
1484             }
1485             String rd = newval != null ? newval.getRd() : null;
1486             if (rd != null) {
1487                 BgpRouter br = getClient(YANG_OBJ);
1488                 if (br == null) {
1489                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1490                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1491                     return;
1492                 }
1493
1494                 for (AddressFamiliesVrf adfvrf : adFamilyVrfToAdd) {
1495                     try {
1496                         LOG.debug("call addVRf rd {} afi {} safi {}", rd, adfvrf.getAfi(), adfvrf.getSafi());
1497                         br.addVrf(BgpUtil.getLayerType(adfvrf), rd, newval.getImportRts(),
1498                                 newval.getExportRts(),adfvrf.getAfi().toJava(), adfvrf.getSafi().toJava());
1499                     } catch (TException | BgpRouterException e) {
1500                         LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1501                     }
1502                 }
1503
1504                 for (AddressFamiliesVrf adfToDel : adFamilyVrfToDel) {
1505                     try {
1506                         LOG.debug("call delVRf rd {} afi {} safi {}", rd, adfToDel.getAfi(), adfToDel.getSafi());
1507                         br.delVrf(rd, adfToDel.getAfi().toJava(), adfToDel.getSafi().toJava());
1508                     } catch (TException | BgpRouterException e) {
1509                         LOG.error("{} delVrf received exception; {}", YANG_OBJ, ADD_WARN, e);
1510                     }
1511                 }
1512             }
1513         }
1514     }
1515
1516     public class BgpReactor
1517             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1518             implements ClusteredDataTreeChangeListener<Bgp> {
1519
1520         private static final String YANG_OBJ = "Bgp ";
1521
1522         public BgpReactor() {
1523             super(Bgp.class, BgpReactor.class);
1524         }
1525
1526
1527         @Override
1528         protected void add(InstanceIdentifier<Bgp> iid, Bgp val) {
1529             LOG.debug("received add Bgp config");
1530
1531             try {
1532                 initer.await();
1533             } catch (InterruptedException e) {
1534                 // Ignored
1535             }
1536             synchronized (BgpConfigurationManager.this) {
1537                 config = val;
1538                 if (!isBGPEntityOwner()) {
1539                     return;
1540                 }
1541             }
1542         }
1543
1544         @Override
1545         protected BgpReactor getDataTreeChangeListener() {
1546             return BgpReactor.this;
1547         }
1548
1549         @Override
1550         protected InstanceIdentifier<Bgp> getWildCardPath() {
1551             return InstanceIdentifier.create(Bgp.class);
1552         }
1553
1554         @Override
1555         protected void remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1556             if (!isBGPEntityOwner()) {
1557                 return;
1558             }
1559             LOG.debug("received remove Bgp config");
1560
1561             config = null;
1562         }
1563
1564         @Override
1565         protected void update(InstanceIdentifier<Bgp> iid,
1566                 Bgp oldval, Bgp newval) {
1567             if (!isBGPEntityOwner()) {
1568                 return;
1569             }
1570
1571             config = newval;
1572         }
1573     }
1574
1575     @SuppressWarnings("deprecation")
1576     public class MultipathReactor
1577             extends AsyncDataTreeChangeListenerBase<Multipath, MultipathReactor>
1578             implements ClusteredDataTreeChangeListener<Multipath> {
1579
1580         private static final String YANG_OBJ = "multipath ";
1581
1582         public MultipathReactor() {
1583             super(Multipath.class, MultipathReactor.class);
1584         }
1585
1586
1587         @Override
1588         protected MultipathReactor getDataTreeChangeListener() {
1589             return MultipathReactor.this;
1590         }
1591
1592         @Override
1593         protected InstanceIdentifier<Multipath> getWildCardPath() {
1594             return InstanceIdentifier.create(Bgp.class).child(MultipathContainer.class).child(Multipath.class);
1595         }
1596
1597         @Override
1598         protected void remove(InstanceIdentifier<Multipath> iid, Multipath val) {
1599             executor.execute(new MultipathStatusChange(val));
1600         }
1601
1602         @Override
1603         protected void update(InstanceIdentifier<Multipath> iid, Multipath oldval, Multipath newval) {
1604             executor.execute(new MultipathStatusChange(newval));
1605         }
1606
1607         @Override
1608         protected void add(InstanceIdentifier<Multipath> key, Multipath dataObjectModification) {
1609             executor.execute(new MultipathStatusChange(dataObjectModification));
1610         }
1611
1612         class MultipathStatusChange implements Runnable {
1613
1614             Multipath multipath;
1615
1616             MultipathStatusChange(Multipath multipath) {
1617                 this.multipath = multipath;
1618             }
1619
1620             @Override
1621             public void run() {
1622                 if (isBGPEntityOwner()) {
1623                     synchronized (BgpConfigurationManager.this) {
1624
1625                         BgpRouter br = getClient(YANG_OBJ);
1626
1627                         if (br != null) {
1628                             af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1629                             af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1630
1631                             try {
1632                                 if (multipath.isMultipathEnabled()) {
1633                                     br.enableMultipath(afi, safi);
1634                                 } else {
1635                                     br.disableMultipath(afi, safi);
1636                                 }
1637                             } catch (TException | BgpRouterException e) {
1638                                 LOG.error("{} received exception", YANG_OBJ, e);
1639                             }
1640                         }
1641                     }
1642                 }
1643             }
1644
1645         }
1646
1647         @Override
1648         public void close() {
1649             super.close();
1650         }
1651     }
1652
1653     @SuppressWarnings("deprecation")
1654     public class VrfMaxpathReactor
1655             extends AsyncDataTreeChangeListenerBase<VrfMaxpath, VrfMaxpathReactor>
1656             implements ClusteredDataTreeChangeListener<VrfMaxpath> {
1657
1658         private static final String YANG_OBJ = "vrfMaxpath ";
1659
1660         public VrfMaxpathReactor() {
1661             super(VrfMaxpath.class, VrfMaxpathReactor.class);
1662         }
1663
1664
1665         @Override
1666         protected VrfMaxpathReactor getDataTreeChangeListener() {
1667             return VrfMaxpathReactor.this;
1668         }
1669
1670         @Override
1671         protected InstanceIdentifier<VrfMaxpath> getWildCardPath() {
1672             return InstanceIdentifier.create(Bgp.class).child(VrfMaxpathContainer.class).child(VrfMaxpath.class);
1673         }
1674
1675         class VrfMaxPathConfigurator implements Runnable {
1676
1677             VrfMaxpath vrfMaxpathVal;
1678
1679             VrfMaxPathConfigurator(VrfMaxpath vrfMaxPathVal) {
1680                 this.vrfMaxpathVal = vrfMaxPathVal;
1681             }
1682
1683             @Override
1684             public void run() {
1685                 if (isBGPEntityOwner()) {
1686                     synchronized (BgpConfigurationManager.this) {
1687                         BgpRouter br = getClient(YANG_OBJ);
1688                         if (br != null) {
1689                             try {
1690                                 br.multipaths(vrfMaxpathVal.getRd(), vrfMaxpathVal.getMaxpaths().toJava());
1691                                 LOG.debug("Maxpath for vrf {} is {}", vrfMaxpathVal.getRd(),
1692                                         vrfMaxpathVal.getMaxpaths());
1693                             } catch (TException | BgpRouterException e) {
1694                                 LOG.error("{} received exception", YANG_OBJ, e);
1695                             }
1696                         }
1697                     }
1698                 }
1699             }
1700         }
1701
1702         @Override
1703         protected void remove(InstanceIdentifier<VrfMaxpath> iid, VrfMaxpath vrfMaxPathVal) {
1704             if (isBGPEntityOwner()) {
1705                 synchronized (BgpConfigurationManager.this) {
1706                     BgpRouter br = getClient(YANG_OBJ);
1707                     if (br != null) {
1708                         try {
1709                             br.multipaths(vrfMaxPathVal.getRd(), BgpConstants.BGP_DEFAULT_MULTIPATH);
1710                             LOG.debug("Del Maxpath for vrf: {} ", vrfMaxPathVal.getRd());
1711                         } catch (TException | BgpRouterException e) {
1712                             LOG.error(YANG_OBJ + " del received exception:", e);
1713                         }
1714                     }
1715                 }
1716             }
1717         }
1718
1719         @Override
1720         protected void update(InstanceIdentifier<VrfMaxpath> iid,
1721                               VrfMaxpath oldval, VrfMaxpath newval) {
1722             if (!Objects.equals(oldval.getMaxpaths(), newval.getMaxpaths())) {
1723                 executor.execute(new VrfMaxPathConfigurator(newval));
1724             }
1725         }
1726
1727         @Override
1728         protected void add(InstanceIdentifier<VrfMaxpath> instanceIdentifier, VrfMaxpath vrfMaxpathVal) {
1729             executor.execute(new VrfMaxPathConfigurator(vrfMaxpathVal));
1730         }
1731
1732         @Override
1733         public void close() {
1734             super.close();
1735         }
1736     }
1737
1738     public class BfdConfigReactor
1739             extends AsyncDataTreeChangeListenerBase<BfdConfig, BfdConfigReactor>
1740             implements ClusteredDataTreeChangeListener<BfdConfig> {
1741
1742         private static final String YANG_OBJ = "BfdConfig ";
1743
1744         public BfdConfigReactor() {
1745             super(BfdConfig.class, BfdConfigReactor.class);
1746         }
1747
1748         @Override
1749         protected void add(InstanceIdentifier<BfdConfig> iid, BfdConfig val) {
1750             if (!isBGPEntityOwner()) {
1751                 return;
1752             }
1753             BgpRouter br = getClient(YANG_OBJ);
1754             LOG.debug("received bfd config: bfd enabled {} min-rx {} min-tx {} detect-mul {} mhop {}",
1755                     val.isBfdEnabled(), val.getMinRx(), val.getMinTx(),
1756                     val.getDetectMult(), val.isMultihop());
1757             if (br == null) {
1758                 LOG.debug(YANG_OBJ + "{} Unable to process add  {}",
1759                         BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1760                 return;
1761             }
1762             if (val.isBfdEnabled() == false) {
1763                 LOG.debug("BFD not enabled. Ignoring the config add");
1764                 return;
1765             }
1766             int minRx = val.getMinRx().intValue();
1767             int minTx = val.getMinTx().intValue();
1768             int detectMult = val.getDetectMult().intValue();
1769             boolean multiHop = val.isMultihop();
1770             try {
1771                 br.addBfd(detectMult, minRx, minTx,multiHop);
1772             } catch (TException | BgpRouterException e) {
1773                 LOG.error("{} get {}, Add received exception;", YANG_OBJ, ADD_WARN, e);
1774             }
1775         }
1776
1777         @Override
1778         protected BfdConfigReactor getDataTreeChangeListener() {
1779             return BfdConfigReactor.this;
1780         }
1781
1782         @Override
1783         protected InstanceIdentifier<BfdConfig> getWildCardPath() {
1784             return InstanceIdentifier.create(BfdConfig.class);
1785         }
1786
1787         @Override
1788         protected void remove(InstanceIdentifier<BfdConfig> iid, BfdConfig val) {
1789             if (!isBGPEntityOwner()) {
1790                 return;
1791             }
1792             LOG.debug("received bfd config removal");
1793             BgpRouter br = getClient(YANG_OBJ);
1794             if (br == null) {
1795                 LOG.debug("{} Unable to process del {}  {}", YANG_OBJ,
1796                         BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1797                 return;
1798             }
1799             try {
1800                 br.delBfd();
1801             } catch (TException | BgpRouterException e) {
1802                 LOG.error("{} get {}, Del received exception;", YANG_OBJ, ADD_WARN, e);
1803             }
1804
1805         }
1806
1807         @Override
1808         protected void update(InstanceIdentifier<BfdConfig> iid,
1809                               BfdConfig oldval, BfdConfig newval) {
1810             LOG.debug("received bfd config: updated oldval bfd enabled {}"
1811                     + "min-rx {} min-tx {} detect-mul {} mhop {}",
1812                     oldval.isBfdEnabled(), oldval.getMinRx(), oldval.getMinTx(),
1813                     oldval.getDetectMult(), oldval.isMultihop());
1814             LOG.debug("received bfd config: updated newval bfd enabled {}"
1815                     + "min-rx {} min-tx {} detect-mul {} mhop {}",
1816                     newval.isBfdEnabled(), newval.getMinRx(), newval.getMinTx(),
1817                     newval.getDetectMult(), newval.isMultihop());
1818             if (oldval.isBfdEnabled()) {
1819                 LOG.debug("deleting bfd config on an update");
1820                 remove(iid, oldval);
1821             }
1822             LOG.debug("adding bfd config on an update");
1823             add(iid, newval);
1824         }
1825     }
1826
1827
1828     public boolean isIpAvailable(String odlip) {
1829
1830         try {
1831             if (odlip != null) {
1832                 if ("127.0.0.1".equals(odlip)) {
1833                     return true;
1834                 }
1835                 Enumeration<NetworkInterface> networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces();
1836                 while (networkInterfaceEnumeration.hasMoreElements()) {
1837                     NetworkInterface networkInterface = networkInterfaceEnumeration.nextElement();
1838                     Enumeration<InetAddress>  inetAddressEnumeration = networkInterface.getInetAddresses();
1839                     while (inetAddressEnumeration.hasMoreElements()) {
1840                         InetAddress inetAddress = inetAddressEnumeration.nextElement();
1841                         if (odlip.equals(inetAddress.getHostAddress())) {
1842                             return true;
1843                         }
1844                     }
1845                 }
1846             }
1847         } catch (SocketException e) {
1848             // Ignored?
1849         }
1850         return false;
1851     }
1852
1853     public long getStalePathtime(int defValue, AsId asId) {
1854         long spt = 0;
1855         try {
1856             InstanceIdentifier<GracefulRestart> id =
1857                     InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
1858             Optional<GracefulRestart> gracefulRestartOptional = MDSALUtil.read(dataBroker,
1859                     LogicalDatastoreType.CONFIGURATION, id);
1860             if (gracefulRestartOptional.isPresent()) {
1861                 spt = gracefulRestartOptional.get().getStalepathTime().toJava();
1862             }
1863         } catch (NullPointerException e) {
1864             try {
1865                 spt = asId.getStalepathTime().toJava();
1866                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1867             } catch (NullPointerException ignore) {
1868                 LOG.trace("BGP AS id is not set using graceful");
1869                 spt = defValue;
1870             }
1871         }
1872         if (spt == 0) {
1873             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1874             spt = defValue;
1875         }
1876         return spt;
1877     }
1878
1879     public static boolean isValidConfigBgpHostPort(String bgpHost, int bgpPort) {
1880         if (!bgpHost.equals(DEF_CHOST)) {
1881             return true;
1882         } else {
1883             return false;
1884         }
1885     }
1886
1887     public synchronized void bgpRestarted() {
1888         /*
1889          * If there a thread which in the process of stale cleanup, cancel it
1890          * and start a new thread (to avoid processing same again).
1891          */
1892         if (previousReplayJobInProgress()) {
1893             cancelPreviousReplayJob();
1894         }
1895         Runnable task = () -> {
1896             try {
1897                 LOG.info("running bgp replay task ");
1898                 if (get() == null) {
1899                     String host = getConfigHost();
1900                     int port = getConfigPort();
1901                     LOG.info("connecting  to bgp host {} ", host);
1902                     bgpRouter.connect(host, port);
1903                     LOG.info("no config to push in bgp replay task ");
1904                     return;
1905                 }
1906                 setStaleStartTime(System.currentTimeMillis());
1907                 LOG.info("started creating stale fibDSWriter  map ");
1908                 createStaleFibMap();
1909                 setStaleEndTime(System.currentTimeMillis());
1910                 LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
1911                 LOG.info("started bgp config replay ");
1912                 setCfgReplayStartTime(System.currentTimeMillis());
1913                 boolean replaySucceded = replay();
1914                 setCfgReplayEndTime(System.currentTimeMillis());
1915                 LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1916                 if (replaySucceded) {
1917                     long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1918                     setStaleCleanupTime(routeSyncTime);
1919                     LOG.error("starting the stale cleanup timer: {} seconds", routeSyncTime);
1920                     routeCleanupFuture = executor.schedule(new RouteCleanup(), routeSyncTime, TimeUnit.SECONDS);
1921                 } else {
1922                     LOG.error("skipping stale cleanup, may be due to exception while replay");
1923                     staledFibEntriesMap.clear();
1924                 }
1925             } catch (InterruptedException | TimeoutException | ExecutionException eCancel) {
1926                 LOG.error("Stale Cleanup Task Cancelled", eCancel);
1927             }
1928         };
1929         lastReplayJobFt = executor.submit(task);
1930     }
1931
1932     private boolean previousReplayJobInProgress() {
1933         return ((lastReplayJobFt != null && !lastReplayJobFt.isDone())
1934                 || (routeCleanupFuture != null && !routeCleanupFuture.isDone()));
1935     }
1936
1937     private void cancelPreviousReplayJob() {
1938         try {
1939             LOG.error("cancelling already running bgp replay task");
1940             if (lastReplayJobFt != null) {
1941                 lastReplayJobFt.cancel(true);
1942                 lastReplayJobFt = null;
1943                 staledFibEntriesMap.clear();
1944             }
1945             if (routeCleanupFuture != null) {
1946                 routeCleanupFuture.cancel(true);
1947                 routeCleanupFuture = null;
1948                 staledFibEntriesMap.clear();
1949             }
1950             Thread.sleep(2000);
1951         } catch (InterruptedException e) {
1952             LOG.error("Failed to cancel previous replay job ", e);
1953         }
1954     }
1955
1956     private void doRouteSync() {
1957         for (af_afi afi : af_afi.values()) {
1958             try {
1959                 bgpRouter.initRibSync(bgpSyncHandle);
1960             } catch (BgpRouterException e) {
1961                 LOG.error("Route sync aborted, exception when initializing", e);
1962                 return;
1963             }
1964             LOG.error("Starting BGP route sync for afi {}", afi.getValue());
1965             while (bgpSyncHandle.getState() != BgpSyncHandle.DONE) {
1966                 Routes routes = null;
1967                 int noUpdates = 0;
1968                 try {
1969                     routes = bgpRouter.doRibSync(bgpSyncHandle, afi);
1970                 } catch (TException | BgpRouterException e) {
1971                     LOG.error("Route sync aborted, exception when syncing", e);
1972                     return;
1973                 }
1974                 Iterator<Update> updates = routes.getUpdatesIterator();
1975                 while (updates.hasNext()) {
1976                     noUpdates++;
1977                     Update update = updates.next();
1978                     String rd = update.getRd();
1979                     String nexthop = update.getNexthop();
1980
1981                     // TODO: decide correct label here
1982                     int label = update.getL3label();
1983                     int l2label = update.getL2label();
1984
1985                     String prefix = update.getPrefix();
1986                     int plen = update.getPrefixlen();
1987
1988
1989                     // TODO: protocol type will not be available in "update"
1990                     // use "rd" to query vrf table and obtain the protocol_type.
1991                     // Currently using PROTOCOL_EVPN as default.
1992                     onUpdatePushRoute(
1993                            protocol_type.PROTOCOL_L3VPN,
1994                            rd,
1995                            prefix,
1996                            plen,
1997                            nexthop,
1998                            update.getMacaddress(),
1999                            Uint32.valueOf(label),
2000                            Uint32.valueOf(l2label),
2001                            update.getRoutermac(),
2002                            afi);
2003                 }
2004                 LOG.error("No of updates for afi {} is {}", afi.getValue(), noUpdates);
2005             }
2006         }
2007         try {
2008             LOG.error("Ending BGP route-sync");
2009             bgpRouter.endRibSync(bgpSyncHandle);
2010         } catch (BgpRouterException e) {
2011             LOG.error("Route sync aborted, exception when ending", e);
2012         }
2013     }
2014
2015     public void addTepToElanDS(String rd, String tepIp, String mac, Uint32 l2vni) {
2016         boolean needUpdate = addToRt2TepMap(rd, tepIp, mac, l2vni);
2017         if (needUpdate) {
2018             LOG.info("Adding tepIp {} with RD {} to ELan DS", tepIp, rd);
2019             bgpUtil.addTepToElanInstance(rd, tepIp);
2020         } else {
2021             LOG.debug("Skipping the Elan update for RT2 from tep {} rd {}", tepIp, rd);
2022         }
2023     }
2024
2025     public void deleteTepfromElanDS(String rd, String tepIp, String mac) {
2026         boolean needUpdate = deleteFromRt2TepMap(rd, tepIp, mac);
2027         if (needUpdate) {
2028             LOG.info("Deleting tepIp {} with RD {} to ELan DS", tepIp, rd);
2029             bgpUtil.deleteTepFromElanInstance(rd, tepIp);
2030         } else {
2031             LOG.debug("Skipping the Elan update for RT2 withdraw from tep {} rd {}", tepIp, rd);
2032         }
2033     }
2034
2035     /* onUpdatePushRoute
2036      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
2037      *  - Entry compare shall include NextHop, Label.
2038      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
2039      *  - If entry not found, add to FIB Config DS.
2040      *  - If entry found, but either Label/NextHop doesn't match.
2041      *      - Update FIB Config DS with modified values.
2042      *      - delete from Stale Map.
2043      */
2044
2045     public void onUpdatePushRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
2046                                   String macaddress, Uint32 label, Uint32 l2label, String routermac, af_afi afi) {
2047         PrefixUpdateEvent prefixUpdateEvent = new PrefixUpdateEvent(protocolType,rd,prefix,plen,nextHop,
2048                 macaddress,label,l2label,routermac,afi);
2049         bgpUpdatesHistory.addToHistory(TransactionType.ADD, prefixUpdateEvent);
2050         boolean addroute = false;
2051         boolean macupdate = false;
2052         Uint32 l3vni = Uint32.ZERO;
2053         VrfEntry.EncapType encapType = VrfEntry.EncapType.Mplsgre;
2054         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
2055             encapType = VrfEntry.EncapType.Vxlan;
2056             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
2057             if (vpnInstanceOpDataEntry != null) {
2058                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
2059                     LOG.info("Got RT2 route for RD {} l3label {} l2label {} from tep {} with mac {} remote RD {}",
2060                             vpnInstanceOpDataEntry.getVpnInstanceName(), label, l2label, nextHop, macaddress, rd);
2061                     addTepToElanDS(rd, nextHop, macaddress, l2label);
2062                     macupdate = true;
2063                 } else {
2064                     l3vni = vpnInstanceOpDataEntry.getL3vni();
2065                 }
2066             } else {
2067                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
2068                 return;
2069             }
2070         }
2071
2072         if (!staledFibEntriesMap.isEmpty()) {
2073             // restart Scenario, as MAP is not empty.
2074             Map<String, Uint32> map = staledFibEntriesMap.get(rd);
2075             if (map != null) {
2076                 String prefixNextHop = appendNextHopToPrefix(prefix + "/" + plen, nextHop);
2077                 Uint32 labelInStaleMap = map.get(prefixNextHop);
2078                 if (null == labelInStaleMap) {
2079                     // New Entry, which happened to be added during restart.
2080                     addroute = true;
2081                 } else {
2082                     map.remove(prefixNextHop);
2083                     if (isRouteModified(label, labelInStaleMap)) {
2084                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
2085                         // Existing entry, where in Label got modified during restart
2086                         addroute = true;
2087                     }
2088                 }
2089             } else {
2090                 LOG.debug("rd {} map is null while processing prefix {} ", rd, prefix);
2091                 addroute = true;
2092             }
2093         } else {
2094             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
2095             addroute = true;
2096         }
2097         if (macupdate) {
2098             LOG.info("ADD: Adding Mac Fib entry rd {} mac{} nexthop {} l2vni {}", rd, macaddress, nextHop, l2label);
2099             fibDSWriter.addMacEntryToDS(rd, macaddress, prefix, Collections.singletonList(nextHop),
2100                     encapType, l2label, routermac, RouteOrigin.BGP);
2101             LOG.info("ADD: Added Mac Fib entry rd {} prefix {} nexthop {} label {}", rd, macaddress, nextHop, l2label);
2102         } else if (addroute) {
2103             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {} afi {}",
2104                     rd, prefix, nextHop, label, afi);
2105             // TODO: modify addFibEntryToDS signature
2106             List<String> nextHopList = Collections.singletonList(nextHop);
2107             fibDSWriter.addFibEntryToDS(rd, prefix + "/" + plen, nextHopList, encapType, label, l3vni,
2108                                         routermac, RouteOrigin.BGP);
2109             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
2110             String vpnName = bgpUtil.getVpnNameFromRd(rd);
2111             if (vpnName != null) {
2112                 vpnLinkService.leakRouteIfNeeded(vpnName, prefix, nextHopList, label, RouteOrigin.BGP,
2113                                                  NwConstants.ADD_FLOW);
2114             }
2115         }
2116     }
2117
2118     public void onUpdateWithdrawRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
2119             String macaddress) {
2120         PrefixWithdrawEvent prefixWithdrawEvent = new PrefixWithdrawEvent(protocolType,rd,prefix,plen,
2121                 nextHop,macaddress);
2122         bgpUpdatesHistory.addToHistory(TransactionType.ADD, prefixWithdrawEvent);
2123         boolean macupdate = false;
2124         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
2125             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
2126             if (vpnInstanceOpDataEntry != null) {
2127                 Uint32 vni = vpnInstanceOpDataEntry.getL3vni();
2128                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
2129                     LOG.debug("Got RT2 withdraw for RD {} {} from tep {} with mac {} remote RD {}",
2130                             vpnInstanceOpDataEntry.getVpnInstanceName(), vni, nextHop, macaddress, rd);
2131                     deleteTepfromElanDS(rd, nextHop, macaddress);
2132                     LOG.debug("For rd {}. skipping fib update", rd);
2133                     macupdate = true;
2134                 }
2135             } else {
2136                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
2137                 return;
2138             }
2139         }
2140         if (macupdate) {
2141             LOG.info("Removing Mac Fib entry rd {} mac{} nexthop {} ", rd, macaddress, nextHop);
2142             fibDSWriter.removeMacEntryFromDS(rd, macaddress);
2143             LOG.info("Removed Mac Fib entry rd {} prefix {} nexthop {} ", rd, macaddress, nextHop);
2144         } else {
2145             LOG.info("REMOVE: Removing Fib entry rd {} prefix {}", rd, prefix);
2146             fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nextHop);
2147             LOG.info("REMOVE: Removed Fib entry rd {} prefix {}", rd, prefix);
2148         }
2149     }
2150
2151     //TODO: below function is for testing purpose with cli
2152     public void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
2153         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
2154         fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nexthop);
2155         String vpnName = bgpUtil.getVpnNameFromRd(rd);
2156         if (vpnName != null) {
2157             vpnLinkService.leakRouteIfNeeded(vpnName, prefix, null /*nextHopList*/, Uint32.ZERO /*INVALID_LABEL*/,
2158                                              RouteOrigin.BGP, NwConstants.DEL_FLOW);
2159         }
2160     }
2161
2162     public void peerDown(String ipAddress, long asNumber) {
2163         PeerDownEvent peerDownEvent = new PeerDownEvent(ipAddress,asNumber);
2164         bgpUpdatesHistory.addToHistory(TransactionType.ADD, peerDownEvent);
2165         List<String> tepIpList = bgpUtil.getDcgwTepConfig(ipAddress);
2166         if (tepIpList == null) {
2167             LOG.error("No Tep IP configured for DCGW {} on a peerDown", ipAddress);
2168             return;
2169         }
2170         tepIpList.forEach(tepIp -> {
2171             bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW);
2172         });
2173     }
2174
2175     public void peerUp(String ipAddress, long asNumber) {
2176         PeerUpEvent peerUpEvent = new PeerUpEvent(ipAddress,asNumber);
2177         bgpUpdatesHistory.addToHistory(TransactionType.ADD, peerUpEvent);
2178         List<String> tepIpList = bgpUtil.getDcgwTepConfig(ipAddress);
2179         if (tepIpList == null) {
2180             LOG.error("No Tep IP configured for DCGW {} on a peerUp", ipAddress);
2181             return;
2182         }
2183         tepIpList.forEach(tepIp -> {
2184             bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW);
2185         });
2186     }
2187
2188     private static boolean isRouteModified(Uint32 label, Uint32 labelInStaleMap) {
2189         return labelInStaleMap != null && !labelInStaleMap.equals(label);
2190     }
2191
2192     static class ReplayNbr {
2193         Neighbors nbr;
2194         boolean shouldRetry = false;
2195
2196         public Neighbors getNbr() {
2197             return nbr;
2198         }
2199
2200         public boolean isShouldRetry() {
2201             return shouldRetry;
2202         }
2203
2204         public void setShouldRetry(boolean retryNbr) {
2205             this.shouldRetry = retryNbr;
2206         }
2207
2208         ReplayNbr(Neighbors nbr, boolean shouldRetry) {
2209             this.nbr = nbr;
2210             this.shouldRetry = shouldRetry;
2211         }
2212     }
2213
2214     private static boolean replayNbrConfig(List<Neighbors> neighbors, BgpRouter br) {
2215         if (neighbors == null || neighbors.isEmpty()) {
2216             LOG.error("Replaying nbr configuration, received NULL list ");
2217             return true;
2218         }
2219
2220         List<ReplayNbr> replayNbrList = new ArrayList<>();
2221         for (Neighbors nbr : neighbors) {
2222             if (nbr != null) {
2223                 replayNbrList.add(new ReplayNbr(nbr, true));
2224             }
2225         }
2226         final int numberOfNbrRetries = 3;
2227         RetryOnException nbrRetry = new RetryOnException(numberOfNbrRetries);
2228         do {
2229             for (ReplayNbr replayNbr : replayNbrList) {
2230                 if (!replayNbr.isShouldRetry()) {
2231                     continue;
2232                 }
2233                 boolean replayDone = false;
2234                 LOG.debug("Replaying addNbr {}", replayNbr.getNbr().getAddress().getValue());
2235                 replayDone = false;
2236                 try {
2237                     final String md5password = extractMd5Secret(replayNbr.getNbr());
2238                     br.addNeighbor(replayNbr.getNbr().getAddress().getValue(),
2239                             replayNbr.getNbr().getRemoteAs().longValue(), md5password);
2240                     replayDone = true;
2241                 } catch (TApplicationException tae) {
2242                     LOG.debug("Replaying addNbr {}, tapplicationexception: ",
2243                             replayNbr.getNbr().getAddress().getValue(), tae);
2244                     if (tae.getType() == BgpRouterException.BGP_ERR_PEER_EXISTS) {
2245                         LOG.debug("Replaying addNbr Neighbor already present");
2246                         replayDone = true;
2247                     } else {
2248                         LOG.error("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), tae);
2249                     }
2250                 } catch (TException | BgpRouterException eNbr) {
2251                     LOG.debug("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), eNbr);
2252                 }
2253
2254                 LOG.debug("Replay addNbr {} successful", replayNbr.getNbr().getAddress().getValue());
2255
2256                 //Update Source handling
2257                 UpdateSource us = replayNbr.getNbr().getUpdateSource();
2258                 if (us != null) {
2259                     LOG.debug("Replaying updatesource {} to peer {}", us.getSourceIp().getValue(),
2260                             us.getPeerIp().getValue());
2261                     try {
2262                         br.addUpdateSource(us.getPeerIp().getValue(),
2263                                 us.getSourceIp().getValue());
2264                     } catch (TException | BgpRouterException eUs) {
2265                         LOG.debug("Replaying UpdateSource for Nbr {}, exception:",
2266                                 replayNbr.getNbr().getAddress().getValue(), eUs);
2267                     }
2268                     LOG.debug("Replay updatesource {} successful", us.getSourceIp().getValue());
2269                 }
2270                 //Ebgp Multihope
2271                 EbgpMultihop en = replayNbr.getNbr().getEbgpMultihop();
2272                 if (en != null) {
2273                     try {
2274                         br.addEbgpMultihop(en.getPeerIp().getValue(),
2275                                 en.getNhops().intValue());
2276                     } catch (TException | BgpRouterException eEbgpMhop) {
2277                         LOG.debug("Replaying EbgpMultihop for Nbr {}, exception: ",
2278                                 replayNbr.getNbr().getAddress().getValue(), eEbgpMhop);
2279                     }
2280                 }
2281
2282                 //afs
2283                 List<AddressFamilies> afs = replayNbr.getNbr().getAddressFamilies();
2284                 if (afs != null) {
2285                     for (AddressFamilies af : afs) {
2286                         af_afi afi = af_afi.findByValue(af.getAfi().intValue());
2287                         af_safi safi = af_safi.findByValue(af.getSafi().intValue());
2288                         try {
2289                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
2290                         } catch (TException | BgpRouterException eAFs) {
2291                             LOG.debug("Replaying AddressFamily for Nbr {}, exception:",
2292                                     replayNbr.getNbr().getAddress().getValue(), eAFs);
2293                         }
2294                     }
2295                 }
2296                 //replay is success --> no need to replay this nbr in next iteration.
2297                 replayNbr.setShouldRetry(replayDone ? false : true);
2298             }
2299         } while (nbrRetry.decrementAndRetry());
2300         boolean replaySuccess = true;
2301         for (ReplayNbr replayNbr : replayNbrList) {
2302             replaySuccess = replaySuccess && !replayNbr.isShouldRetry();
2303             if (replaySuccess == false) {
2304                 LOG.error("replayNbrConfig: will be cancelling stale cleanup, cfg nbr: {} Failed:",
2305                         replayNbr.getNbr().getAddress().getValue());
2306             }
2307         }
2308         return replaySuccess;
2309     }
2310
2311     public String getConfigHost() {
2312         if (config == null) {
2313             return hostStartup;
2314         }
2315         ConfigServer ts = config.getConfigServer();
2316         return ts == null ? hostStartup : ts.getHost().getValue();
2317     }
2318
2319     public int getConfigPort() {
2320         if (config == null) {
2321             return Integer.parseInt(portStartup);
2322         }
2323         ConfigServer ts = config.getConfigServer();
2324         return ts == null ? Integer.parseInt(portStartup) :
2325                 ts.getPort().intValue();
2326     }
2327
2328     public Bgp getConfig() {
2329         AtomicInteger bgpDSretryCount = new AtomicInteger(DS_RETRY_COUNT);
2330         while (0 != bgpDSretryCount.decrementAndGet()) {
2331             try {
2332                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
2333                         InstanceIdentifier.create(Bgp.class)).orNull();
2334             } catch (ReadFailedException e) {
2335                 //Config DS may not be up, so sleep for 1 second and retry
2336                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
2337                 try {
2338                     Thread.sleep(WAIT_TIME_BETWEEN_EACH_TRY_MILLIS);
2339                 } catch (InterruptedException timerEx) {
2340                     LOG.debug("WAIT_TIME_BETWEEN_EACH_TRY_MILLIS, Timer got interrupted while waiting for"
2341                             + "config DS availability", timerEx);
2342                 }
2343             }
2344         }
2345         LOG.error("failed to get bgp config");
2346         return null;
2347     }
2348
2349     @SuppressWarnings("checkstyle:IllegalCatch")
2350     public synchronized boolean replay() throws InterruptedException, TimeoutException, ExecutionException {
2351         boolean replaySucceded = true;
2352         boolean doRouteSync = false;
2353         String host = getConfigHost();
2354         int port = getConfigPort();
2355         LOG.error("connecting  to bgp host {} ", host);
2356         boolean res = bgpRouter.connect(host, port);
2357         if (!res) {
2358             LOG.error("Cannot connect to BGP config server at {} {}", host, port);
2359             return replaySucceded;
2360         }
2361         config = getConfig();
2362         if (config == null) {
2363             LOG.error("bgp config is empty nothing to push to bgp");
2364             return replaySucceded;
2365         }
2366         BgpRouter br = bgpRouter;
2367         AsId asId = config.getAsId();
2368         if (asId == null) {
2369             LOG.error("bgp as-id is null");
2370             return replaySucceded;
2371         }
2372         long asNum = asId.getLocalAs().toJava();
2373         IpAddress routerId = asId.getRouterId();
2374         String rid = routerId == null ? "" : routerId.stringValue();
2375         int stalepathTime = (int) getStalePathtime(bgpGrRestartTime, config.getAsId());
2376         boolean announceFbit = true;
2377         boolean replayDone = false;
2378         final int numberOfStartBgpRetries = 3;
2379         RetryOnException startBgpRetry = new RetryOnException(numberOfStartBgpRetries);
2380         do {
2381             try {
2382                 LOG.debug("Replaying BGPConfig ");
2383                 br.startBgp(asNum, rid, bgpKaTime, bgpHoldTime, stalepathTime, announceFbit);
2384                 LOG.debug("Replay BGPConfig successful");
2385                 replayDone = true;
2386                 break;
2387             } catch (BgpRouterException bre) {
2388                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
2389                     LOG.debug("Starting the routesync for exception", bre);
2390                     startBgpRetry.errorOccured();
2391                     if (!startBgpRetry.shouldRetry()) {
2392                         doRouteSync = true;
2393                         replayDone = true;
2394                     }
2395                 } else {
2396                     LOG.error("Replay: startBgp() received exception error {} : ",
2397                             bre.getErrorCode(), bre);
2398                     startBgpRetry.errorOccured();
2399                 }
2400             } catch (TApplicationException tae) {
2401                 if (tae.getType() == BgpRouterException.BGP_ERR_ACTIVE) {
2402                     LOG.debug("Starting the routesync for exception", tae);
2403                     startBgpRetry.errorOccured();
2404                     if (!startBgpRetry.shouldRetry()) {
2405                         doRouteSync = true;
2406                         replayDone = true;
2407                     }
2408                 } else if (tae.getType() == BgpRouterException.BGP_ERR_COMMON_FAILURE) {
2409                     LOG.debug("Starting the routesync for AS-ID started exception", tae);
2410                     startBgpRetry.errorOccured();
2411                     if (!startBgpRetry.shouldRetry()) {
2412                         doRouteSync = true;
2413                         replayDone = true;
2414                     }
2415                 } else {
2416                     LOG.error("Replay: startBgp() received exception type {}: ",
2417                             tae.getType(), tae);
2418                     startBgpRetry.errorOccured();
2419                 }
2420             } catch (Exception e) {
2421                 //not unusual. We may have restarted & BGP is already on
2422                 LOG.error("Replay:startBgp() received exception: ", e);
2423                 startBgpRetry.errorOccured();
2424             }
2425         } while (startBgpRetry.shouldRetry());
2426
2427         replaySucceded = replayDone;
2428
2429         startBgpCountersTask();
2430         startBgpAlarmsTask();
2431
2432         /*
2433          * commenting this due to a bug with QBGP. Will uncomment once QBGP fix is done.
2434          * This wont have any functional impacts
2435          */
2436         //try {
2437         //    br.delayEOR(delayEorSeconds);
2438         //} catch (TException | BgpRouterException e) {
2439         //    LOG.error("Replay: delayEOR() number of seconds to wait for EOR from ODL:", e);
2440         //}
2441
2442         BfdConfig bfdConfig = bgpUtil.getBfdConfig();
2443         if (bfdConfig != null) {
2444             if (bfdConfig.isBfdEnabled()) {
2445                 LOG.debug("Replaying bfd config min-rx {} min-tx {} detect-mul {} mhop {}",
2446                         bfdConfig.getMinRx(), bfdConfig.getMinTx(),
2447                         bfdConfig.getDetectMult(), bfdConfig.isMultihop());
2448                 try {
2449                     br.addBfd(bfdConfig.getDetectMult().intValue(), bfdConfig.getMinRx().intValue(),
2450                             bfdConfig.getMinTx().intValue(), bfdConfig.isMultihop());
2451                 } catch (TApplicationException tae) {
2452                     if (tae.getType() == BgpRouterException.BGP_ERR_PEER_EXISTS) {
2453                         LOG.debug("Replay:addBfd() received exception", tae);
2454                     } else {
2455                         LOG.error("Replay:addBfd() received exception", tae);
2456                     }
2457                 } catch (TException | BgpRouterException e) {
2458                     LOG.error("Replay:addBfd() received exception", e);
2459                 }
2460             }
2461         }
2462
2463         List<Neighbors> neighbors = config.getNeighborsContainer() == null ? null
2464                 : config.getNeighborsContainer().getNeighbors();
2465         if (neighbors != null) {
2466             LOG.error("configuring existing Neighbors present for replay total neighbors {}", neighbors.size());
2467             boolean neighborConfigReplayResult = replayNbrConfig(neighbors, br);
2468             if (neighborConfigReplayResult == false) {
2469                 replaySucceded = false;
2470             }
2471         } else {
2472             LOG.error("no Neighbors present for replay config ");
2473         }
2474
2475         Logging logging = config.getLogging();
2476         if (logging != null) {
2477             try {
2478                 br.setLogging(logging.getFile(), logging.getLevel());
2479             } catch (TException | BgpRouterException e) {
2480                 LOG.error("Replay:setLogging() received exception", e);
2481             }
2482         }
2483
2484         GracefulRestart gracefulRestart = config.getGracefulRestart();
2485         bgpGrRestartTime = ((gracefulRestart != null)
2486                 ? gracefulRestart.getStalepathTime().intValue() : bgpGrRestartTime);
2487         try {
2488             br.addGracefulRestart(bgpGrRestartTime);
2489         } catch (Exception e) {
2490             LOG.error("Replay:addGr() received exception: ", e);
2491         }
2492         List<Vrfs> vrfs = config.getVrfsContainer() == null ? null
2493                 : config.getVrfsContainer().getVrfs();
2494         if (vrfs == null) {
2495             vrfs = new ArrayList<>();
2496         }
2497         for (Vrfs vrf : vrfs) {
2498             for (AddressFamiliesVrf adf : vrf.getAddressFamiliesVrf()) {
2499                 try {
2500                     br.addVrf(BgpUtil.getLayerType(adf), vrf.getRd(), vrf.getImportRts(),
2501                             vrf.getExportRts(), adf.getAfi().toJava(), adf.getSafi().toJava());
2502                 } catch (TException | BgpRouterException e) {
2503                     LOG.error("Replay:addVrf() received exception", e);
2504                 }
2505             }
2506         }
2507
2508
2509         List<Networks> ln = config.getNetworksContainer() == null ? null
2510                 : config.getNetworksContainer().getNetworks();
2511         if (ln != null) {
2512             for (Networks net : ln) {
2513                 String rd = net.getRd();
2514                 String pfxlen = net.getPrefixLen();
2515                 String nh = net.getNexthop().getValue();
2516                 Long label = net.getLabel().toJava();
2517                 int lbl = label == null ? 0 : label.intValue();
2518                 int l3vni = net.getL3vni() == null ? 0 : net.getL3vni().intValue();
2519                 int l2vni = net.getL2vni() == null ? 0 : net.getL2vni().intValue();
2520                 if (rd == null && lbl > 0) {
2521                     //LU prefix is being deleted.
2522                     rd = Integer.toString(lbl);
2523                 }
2524
2525                 BgpControlPlaneType protocolType = net.getBgpControlPlaneType();
2526                 int ethernetTag = net.getEthtag().intValue();
2527                 String esi = net.getEsi();
2528                 String macaddress = net.getMacaddress();
2529                 EncapType encapType = net.getEncapType();
2530                 String routerMac = net.getRoutermac();
2531
2532                 try {
2533                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni,
2534                             BgpUtil.convertToThriftProtocolType(protocolType),
2535                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
2536                 } catch (TException | BgpRouterException e) {
2537                     LOG.error("Replay:addPfx() received exception", e);
2538                 }
2539             }
2540         }
2541
2542
2543         List<Multipath> multipaths = config.getMultipathContainer() == null ? null
2544                 : config.getMultipathContainer().getMultipath();
2545
2546         if (multipaths != null) {
2547             for (Multipath multipath : multipaths) {
2548                 if (multipath != null) {
2549                     af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
2550                     af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
2551
2552                     try {
2553                         if (multipath.isMultipathEnabled()) {
2554                             br.enableMultipath(afi, safi);
2555                         } else {
2556                             br.disableMultipath(afi, safi);
2557                         }
2558                     } catch (TException | BgpRouterException e) {
2559                         LOG.info("Replay:multipaths() received exception", e);
2560                     }
2561                 }
2562             }
2563         }
2564         List<VrfMaxpath> vrfMaxpaths = config.getVrfMaxpathContainer() == null ? null
2565                 : config.getVrfMaxpathContainer().getVrfMaxpath();
2566         if (vrfMaxpaths != null) {
2567             for (VrfMaxpath vrfMaxpath : vrfMaxpaths) {
2568                 try {
2569                     br.multipaths(vrfMaxpath.getRd(), vrfMaxpath.getMaxpaths().toJava());
2570                 } catch (TException | BgpRouterException e) {
2571                     LOG.info("Replay:vrfMaxPath() received exception", e);
2572                 }
2573             }
2574         }
2575
2576         //send End of Rib Marker to Qthriftd.
2577         final int numberOfEORRetries = 3;
2578         RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
2579         do {
2580             try {
2581                 br.sendEOR();
2582                 LOG.debug("Replay sendEOR() successful");
2583                 break;
2584             } catch (Exception e) {
2585                 eorRetry.errorOccured();
2586                 LOG.error("Replay:sedEOR() received exception:", e);
2587             }
2588         } while (eorRetry.shouldRetry());
2589
2590         if (doRouteSync) {
2591             LOG.debug("starting route sync for Thrift BGP_ERR_COMMON_FAILURE exception "
2592                       + "happened earlier");
2593             doRouteSync();
2594         }
2595
2596         return replaySucceded;
2597     }
2598
2599     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
2600         bgpUtil.update(iid, dto);
2601     }
2602
2603     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
2604         bgpUtil.delete(iid);
2605     }
2606
2607     public void startConfig(String bgpHost, int thriftPort) {
2608         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2609                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2610         InstanceIdentifier<ConfigServer> iid = iib.build();
2611         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
2612         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
2613                 .setPort((long) thriftPort).build();
2614         update(iid, dto);
2615     }
2616
2617     public void startBgp(long as, String routerId, int spt, boolean fbit) {
2618         IpAddress rid = routerId == null ? null : IpAddressBuilder.getDefaultInstance(routerId);
2619         Long staleTime = (long) spt;
2620         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2621                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2622         InstanceIdentifier<AsId> iid = iib.build();
2623         AsId dto = new AsIdBuilder().setLocalAs(as)
2624                 .setRouterId(rid)
2625                 .setStalepathTime(staleTime)
2626                 .setAnnounceFbit(fbit).build();
2627         update(iid, dto);
2628     }
2629
2630     public void startBfd(long detectMult, long minRx, long minTx, boolean multiHop) {
2631         InstanceIdentifier.InstanceIdentifierBuilder<BfdConfig> iib =
2632                 InstanceIdentifier.builder(BfdConfig.class);
2633         InstanceIdentifier<BfdConfig> iid = iib.build();
2634         BfdConfig dto = new BfdConfigBuilder()
2635                 .setBfdEnabled(true)
2636                 .setMultihop(multiHop)
2637                 .setMinRx(minRx)
2638                 .setMinTx(minTx)
2639                 .setDetectMult(detectMult)
2640                 .build();
2641         update(iid, dto);
2642     }
2643
2644     public void addDcgwTep(String dcgwIp, String tepIp) {
2645         InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2646                 InstanceIdentifier.builder(Bgp.class)
2647                         .child(DcgwTepList.class)
2648                         .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2649         InstanceIdentifier<DcgwTep> iid = iib.build();
2650         ArrayList<String> tepList = new ArrayList<String>();
2651         tepList.add(tepIp);
2652         DcgwTep dto = new DcgwTepBuilder().setDcGwIp(dcgwIp).setTepIps(tepList)
2653                 .build();
2654         update(iid, dto);
2655         bgpUtil.removeOrUpdateLBGroups(tepIp,NwConstants.MOD_FLOW);
2656     }
2657
2658     public void addLogging(String fileName, String logLevel) {
2659         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2660                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2661         InstanceIdentifier<Logging> iid = iib.build();
2662         Logging dto = new LoggingBuilder().setFile(fileName)
2663                 .setLevel(logLevel).build();
2664         update(iid, dto);
2665     }
2666
2667     public void addGracefulRestart(int staleTime) {
2668         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2669                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
2670         InstanceIdentifier<GracefulRestart> iid = iib.build();
2671         GracefulRestart dto = new GracefulRestartBuilder()
2672                 .setStalepathTime((long) staleTime).build();
2673         update(iid, dto);
2674     }
2675
2676     public void addNeighbor(
2677             String nbrIp, long remoteAs, @Nullable final TcpMd5SignaturePasswordType md5Secret) {
2678         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2679         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2680                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2681                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2682         InstanceIdentifier<Neighbors> iid = iib.build();
2683         TcpSecurityOption tcpSecOption = null;
2684         if (md5Secret != null) {
2685             tcpSecOption = new TcpMd5SignatureOptionBuilder().setTcpMd5SignaturePassword(md5Secret).build();
2686         } // else let tcpSecOption be null
2687         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
2688                 .setRemoteAs(remoteAs).setTcpSecurityOption(tcpSecOption).build();
2689         update(iid, dto);
2690     } // public addNeighbor(nbrIp, remoteAs, md5Secret)
2691
2692     public void addUpdateSource(String nbrIp, String srcIp) {
2693         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2694         Ipv4Address srcAddr = new Ipv4Address(srcIp);
2695         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2696                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2697                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2698                         .child(UpdateSource.class);
2699         InstanceIdentifier<UpdateSource> iid = iib.build();
2700         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
2701                 .setSourceIp(srcAddr).build();
2702         update(iid, dto);
2703     }
2704
2705     public void addEbgpMultihop(String nbrIp, int hops) {
2706         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2707         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2708                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2709                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2710                         .child(EbgpMultihop.class);
2711         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2712         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
2713                 .setNhops((long) hops).build();
2714         update(iid, dto);
2715     }
2716
2717     public void addAddressFamily(String nbrIp, int afi, int safi) {
2718         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2719         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2720                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2721                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2722                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2723         InstanceIdentifier<AddressFamilies> iid = iib.build();
2724         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
2725                 .setAfi((long) afi).setSafi((long) safi).build();
2726         update(iid, dto);
2727     }
2728
2729     public void addPrefix(String rd, String macAddress, String pfx, List<String> nhList,
2730               VrfEntry.EncapType encapType, Uint32 lbl, Uint32 l3vni, Uint32 l2vni, String gatewayMac) {
2731         for (String nh : nhList) {
2732             Ipv4Address nexthop = nh != null ? new Ipv4Address(nh) : null;
2733             Uint32 label = lbl;
2734             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
2735                     .child(NetworksContainer.class)
2736                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
2737             NetworksBuilder networksBuilder = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
2738                                                 .setLabel(label).setEthtag(BgpConstants.DEFAULT_ETH_TAG);
2739             buildVpnEncapSpecificInfo(networksBuilder, encapType, label, l3vni, l2vni, macAddress, gatewayMac);
2740             update(iid, networksBuilder.build());
2741         }
2742     }
2743
2744     private static void buildVpnEncapSpecificInfo(NetworksBuilder builder, VrfEntry.EncapType encapType, Uint32 label,
2745                                                   Uint32 l3vni, Uint32 l2vni, String macAddress, String gatewayMac) {
2746         if (encapType.equals(VrfEntry.EncapType.Mplsgre)) {
2747             builder.setLabel(label).setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLL3VPN)
2748                     .setEncapType(EncapType.GRE);
2749         } else {
2750             builder.setL3vni(l3vni).setL2vni(l2vni).setMacaddress(macAddress).setRoutermac(gatewayMac)
2751                     .setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLEVPN).setEncapType(EncapType.VXLAN);
2752         }
2753     }
2754
2755     // TODO: add LayerType as arg - supports command
2756     public void addVrf(String rd, List<String> irts, List<String> erts, AddressFamily addressFamily) {
2757         Vrfs vrf = bgpUtil.getVrfFromRd(rd);
2758         List<AddressFamiliesVrf> adfList = new ArrayList<>(1);
2759         if (vrf != null) {
2760             adfList = vrf.getAddressFamiliesVrf();
2761         }
2762         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2763         if (addressFamily.equals(AddressFamily.IPV4)) {
2764             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2765             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2766         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2767             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2768             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2769         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2770             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2771             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2772         }
2773         AddressFamiliesVrf adf = adfBuilder.build();
2774         adfList.add(adf);
2775         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib = InstanceIdentifier.builder(Bgp.class)
2776                 .child(VrfsContainer.class)
2777                 .child(Vrfs.class, new VrfsKey(rd));
2778         InstanceIdentifier<Vrfs> iid = iib.build();
2779         Vrfs dto = new VrfsBuilder().setRd(rd).setImportRts(irts)
2780             .setExportRts(erts).setAddressFamiliesVrf(adfList).build();
2781
2782         List<AddressFamiliesVrf> listAdFamilies = mapNewAdFamily.get(rd);
2783         if (listAdFamilies != null) {
2784             listAdFamilies.add(adf);
2785         } else {
2786             mapNewAdFamily.put(rd, adfList);
2787         }
2788
2789         try {
2790             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
2791         } catch (TransactionCommitFailedException e) {
2792             LOG.error("Error adding VRF to datastore", e);
2793             throw new RuntimeException(e);
2794         }
2795
2796         // enable multipath by default in all VRFs
2797         setMultipaths(rd, BgpConstants.BGP_DEFAULT_MULTIPATH);
2798     }
2799
2800     public void stopConfig() {
2801         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2802                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2803         InstanceIdentifier<ConfigServer> iid = iib.build();
2804         delete(iid);
2805     }
2806
2807     public void stopBgp() {
2808         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2809                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2810         InstanceIdentifier<AsId> iid = iib.build();
2811         delete(iid);
2812     }
2813
2814     public void stopBfd() {
2815         InstanceIdentifier.InstanceIdentifierBuilder<BfdConfig> iib =
2816                 InstanceIdentifier.builder(BfdConfig.class);
2817         InstanceIdentifier<BfdConfig> iid = iib.build();
2818         delete(iid);
2819     }
2820
2821     public void delDcgwTep(String dcgwIp, String tepIp) {
2822         if (tepIp == null) {
2823             InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2824                     InstanceIdentifier.builder(Bgp.class)
2825                             .child(DcgwTepList.class)
2826                             .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2827             InstanceIdentifier<DcgwTep> iid = iib.build();
2828             delete(iid);
2829         } else {
2830             InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2831                     InstanceIdentifier.builder(Bgp.class)
2832                             .child(DcgwTepList.class)
2833                             .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2834             InstanceIdentifier<DcgwTep> iid = iib.build();
2835             List<String> tepIpList = bgpUtil.getDcgwTepConfig(dcgwIp);
2836             if (tepIpList == null) {
2837                 LOG.error("No Tep IP configured for DCGW {} on deleting the dcgwtep", dcgwIp);
2838                 return;
2839             }
2840             List<String> newTepIpList = new ArrayList<String>();
2841             tepIpList.forEach(tep -> {
2842                 if (!tep.equals(tepIp)) {
2843                     newTepIpList.add(tep);
2844                 }
2845             });
2846             DcgwTep dto = new DcgwTepBuilder().setDcGwIp(dcgwIp).setTepIps(newTepIpList)
2847                     .build();
2848             try {
2849                 SingleTransactionDataBroker.syncWrite(dataBroker,
2850                         LogicalDatastoreType.CONFIGURATION, iid, dto);
2851             } catch (TransactionCommitFailedException e) {
2852                 LOG.error("delDcgwTep: Error deleting DCGW Tep", e);
2853                 throw new RuntimeException(e);
2854             }
2855         }
2856     }
2857
2858     public void delLogging() {
2859         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2860                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2861         InstanceIdentifier<Logging> iid = iib.build();
2862         delete(iid);
2863     }
2864
2865     public void delGracefulRestart() {
2866         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2867                 InstanceIdentifier.builder(Bgp.class)
2868                         .child(GracefulRestart.class);
2869         InstanceIdentifier<GracefulRestart> iid = iib.build();
2870         delete(iid);
2871     }
2872
2873     public void delNeighbor(String nbrIp) {
2874         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2875         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2876                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2877                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2878         InstanceIdentifier<Neighbors> iid = iib.build();
2879         delete(iid);
2880     }
2881
2882     public void delUpdateSource(String nbrIp) {
2883         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2884         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2885                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2886                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2887                         .child(UpdateSource.class);
2888         InstanceIdentifier<UpdateSource> iid = iib.build();
2889         delete(iid);
2890     }
2891
2892     public void delEbgpMultihop(String nbrIp) {
2893         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2894         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2895                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2896                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2897                         .child(EbgpMultihop.class);
2898         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2899         delete(iid);
2900     }
2901
2902     public void delAddressFamily(String nbrIp, int afi, int safi) {
2903         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2904         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2905                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2906                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2907                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2908         InstanceIdentifier<AddressFamilies> iid = iib.build();
2909         delete(iid);
2910     }
2911
2912     public void delPrefix(String rd, String pfx) {
2913         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
2914                 InstanceIdentifier.builder(Bgp.class).child(NetworksContainer.class)
2915                         .child(Networks.class, new NetworksKey(pfx, rd));
2916         InstanceIdentifier<Networks> iid = iib.build();
2917         delete(iid);
2918     }
2919
2920     public boolean delVrf(String rd, AddressFamily addressFamily) {
2921         if (addressFamily == null) {
2922             LOG.error("delVrf: vrf {}, addressFamily invalid", rd);
2923             return false;
2924         }
2925         delMultipaths(rd);
2926         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2927         if (addressFamily.equals(AddressFamily.IPV4)) {
2928             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2929             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2930         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2931             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2932             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2933         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2934             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2935             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2936         }
2937         LOG.debug("delVrf: Received Delete VRF : rd:{}, address family: {} {}", rd,
2938                 adfBuilder.getAfi(), adfBuilder.getSafi());
2939
2940         Vrfs vrfOriginal = bgpUtil.getVrfFromRd(rd);
2941         if (vrfOriginal == null) {
2942             LOG.error("delVrf: no vrf with existing rd {}. step aborted", rd);
2943             return false;
2944         }
2945
2946         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2947                 InstanceIdentifier.builder(Bgp.class).child(VrfsContainer.class)
2948                         .child(Vrfs.class, new VrfsKey(rd));
2949
2950         InstanceIdentifier<Vrfs> iid = iib.build();
2951
2952         @SuppressWarnings("static-access")
2953         InstanceIdentifier<Bgp> iid6 =  iid.builder(Bgp.class).build()
2954                 .child(MultipathContainer.class)
2955                 .child(Multipath.class, new MultipathKey(adfBuilder.getAfi(), adfBuilder.getSafi())).create(Bgp.class);
2956         InstanceIdentifierBuilder<Vrfs> iib3 =
2957                 iid6.child(VrfsContainer.class).child(Vrfs.class, new VrfsKey(rd)).builder();
2958         InstanceIdentifier<Vrfs> iidFinal = iib3.build();
2959
2960         //** update or delete the vrfs with the rest of AddressFamilies already present in the last list
2961         AddressFamiliesVrf adfToDel = adfBuilder.build();
2962         List<AddressFamiliesVrf> adfListOriginal = new ArrayList<>(vrfOriginal.nonnullAddressFamiliesVrf());
2963         List<AddressFamiliesVrf> adfListToRemoveFromOriginal = new ArrayList<>();
2964         adfListOriginal.forEach(adf -> {
2965             if (adf.equals(adfToDel)) {
2966                 adfListToRemoveFromOriginal.add(adfToDel);
2967                 return;
2968             }
2969         });
2970         for (AddressFamiliesVrf adfToRemove : adfListToRemoveFromOriginal) {
2971             adfListOriginal.remove(adfToRemove);
2972             try {
2973                 SingleTransactionDataBroker.syncWrite(dataBroker,
2974                         LogicalDatastoreType.CONFIGURATION, iid, vrfOriginal);
2975             } catch (TransactionCommitFailedException e) {
2976                 LOG.error("delVrf: Error updating VRF to datastore", e);
2977                 throw new RuntimeException(e);
2978             }
2979         }
2980         if (adfListOriginal.isEmpty()) {
2981             LOG.debug("delVrf: delete iid: {}", iidFinal);
2982             delete(iidFinal);
2983             return true;
2984         }
2985         // not all is removed
2986         return false;
2987     }
2988
2989     public void setMultipathStatus(af_afi afi, af_safi safi, boolean enable) {
2990         long lafi = afi.getValue();
2991         long lsafi = safi.getValue();
2992
2993         InstanceIdentifier.InstanceIdentifierBuilder<Multipath> iib =
2994                 InstanceIdentifier
2995                         .builder(Bgp.class).child(MultipathContainer.class)
2996                         .child(Multipath.class,
2997                                 new MultipathKey(Long.valueOf(afi.getValue()), Long.valueOf(safi.getValue())));
2998
2999         Multipath dto = new MultipathBuilder().setAfi(lafi).setSafi(lsafi).setMultipathEnabled(enable).build();
3000         update(iib.build(), dto);
3001     }
3002
3003     public void setMultipaths(String rd, int maxpath) {
3004         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
3005                 InstanceIdentifier
3006                         .builder(Bgp.class).child(VrfMaxpathContainer.class)
3007                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
3008
3009         VrfMaxpath dto = new VrfMaxpathBuilder().setRd(rd).setMaxpaths(maxpath).build();
3010         update(iib.build(), dto);
3011     }
3012
3013     public void delMultipaths(String rd) {
3014         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
3015                 InstanceIdentifier.builder(Bgp.class).child(VrfMaxpathContainer.class)
3016                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
3017         InstanceIdentifier<VrfMaxpath> iid = iib.build();
3018         delete(iid);
3019     }
3020
3021     /*
3022     * Remove Stale Marked Routes after timer expiry.
3023     */
3024     private class RouteCleanup implements Callable<Integer> {
3025
3026         @Override
3027         public Integer call() {
3028             totalCleared = 0;
3029             try {
3030                 if (staledFibEntriesMap.isEmpty()) {
3031                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
3032                 } else {
3033                     for (String rd : staledFibEntriesMap.keySet()) {
3034                         if (Thread.interrupted()) {
3035                             return 0;
3036                         }
3037                         Map<String, Uint32> map = staledFibEntriesMap.get(rd);
3038                         if (map != null) {
3039                             for (String key : map.keySet()) {
3040                                 if (Thread.interrupted()) {
3041                                     return 0;
3042                                 }
3043                                 String prefix = extractPrefix(key);
3044                                 String nextHop = extractNextHop(key);
3045                                 totalCleared++;
3046                                 LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}, nextHop:{}",
3047                                         rd, prefix, nextHop);
3048                                 fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix, nextHop);
3049                             }
3050                         }
3051                     }
3052                 }
3053             } finally {
3054                 staledFibEntriesMap.clear();
3055             }
3056             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
3057             return 0;
3058         }
3059     }
3060
3061     /*
3062      * BGP restart scenario, ODL-BGP manager was/is running.
3063      * On re-sync notification, Get a copy of FIB database.
3064      */
3065     public void createStaleFibMap() {
3066         totalStaledCount = 0;
3067         try {
3068             staledFibEntriesMap.clear();
3069             fibDSWriter.clearFibMap();
3070             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
3071
3072             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
3073                     LogicalDatastoreType.CONFIGURATION, id);
3074             if (fibEntries.isPresent()) {
3075                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
3076                 for (VrfTables vrfTable : staleVrfTables) {
3077                     Map<String, Uint32> staleFibEntMap = new HashMap<>();
3078                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
3079                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
3080                             //Stale marking and cleanup is only meant for the routes learned through BGP.
3081                             continue;
3082                         }
3083                         if (Thread.interrupted()) {
3084                             break;
3085                         }
3086                         totalStaledCount++;
3087                         //Create MAP from staleVrfTables.
3088                         vrfEntry.getRoutePaths()
3089                                 .forEach(
3090                                     routePath -> {
3091                                         staleFibEntMap.put(
3092                                                 appendNextHopToPrefix(vrfEntry.getDestPrefix(),
3093                                                         routePath.getNexthopAddress()), routePath.getLabel());
3094                                         fibDSWriter.addEntryToFibMap(
3095                                                 vrfTable.getRouteDistinguisher(),  vrfEntry.getDestPrefix(),
3096                                                 routePath.getNexthopAddress());
3097
3098                                     });
3099                     }
3100                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), staleFibEntMap);
3101                 }
3102             } else {
3103                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
3104             }
3105         } catch (ReadFailedException e) {
3106             LOG.error("createStaleFibMap:: error ", e);
3107         }
3108         LOG.error("created {} staled entries ", totalStaledCount);
3109     }
3110
3111     /*
3112      * BGP config remove scenario, Need to remove all the
3113      * external routes from FIB.
3114      */
3115     public void deleteExternalFibRoutes() {
3116         totalExternalRoutes = 0;
3117         totalExternalMacRoutes = 0;
3118         try {
3119             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
3120
3121             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
3122                     LogicalDatastoreType.CONFIGURATION, id);
3123             if (fibEntries.isPresent()) {
3124                 if (fibEntries.get().getVrfTables() == null) {
3125                     LOG.error("deleteExternalFibRoutes::getVrfTables is null");
3126                     return;
3127                 }
3128                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
3129                 for (VrfTables vrfTable : staleVrfTables) {
3130                     String rd = vrfTable.getRouteDistinguisher();
3131                     if (vrfTable.getVrfEntry() != null) {
3132                         for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
3133                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
3134                                 //route cleanup is only meant for the routes learned through BGP.
3135                                 continue;
3136                             }
3137                             totalExternalRoutes++;
3138                             fibDSWriter.removeFibEntryFromDS(rd, vrfEntry.getDestPrefix());
3139                         }
3140                     } else if (vrfTable.getMacVrfEntry() != null) {
3141                         for (MacVrfEntry macEntry : vrfTable.getMacVrfEntry()) {
3142                             if (RouteOrigin.value(macEntry.getOrigin()) != RouteOrigin.BGP) {
3143                                 //route cleanup is only meant for the routes learned through BGP.
3144                                 continue;
3145                             }
3146                             totalExternalMacRoutes++;
3147                             fibDSWriter.removeMacEntryFromDS(rd, macEntry.getMac());
3148                         }
3149                     }
3150                 }
3151             } else {
3152                 LOG.error("deleteExternalFibRoutes:: FIBentries.class is not present");
3153             }
3154         } catch (ReadFailedException e) {
3155             LOG.error("deleteExternalFibRoutes:: error ", e);
3156         }
3157         LOG.debug("deleted {} fib entries {} mac entries", totalExternalRoutes, totalExternalMacRoutes);
3158     }
3159
3160     public boolean addToRt2TepMap(String rd, String tepIp, String mac, Uint32 l2vni) {
3161         boolean isFirstMacUpdateFromTep = false;
3162         if (rt2TepMap.containsKey(rd)) {
3163             if (rt2TepMap.get(rd).containsKey(tepIp)) {
3164                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and tep-ip {}. No Elan DS write required",
3165                         mac, l2vni, rd, tepIp);
3166                 rt2TepMap.get(rd).get(tepIp).put(mac, l2vni);
3167             } else {
3168                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and new tep-ip {}",
3169                         mac, l2vni, rd, tepIp);
3170                 isFirstMacUpdateFromTep = true;
3171                 Map<String, Uint32> macList = new HashMap<>();
3172                 macList.put(mac, l2vni);
3173                 rt2TepMap.get(rd).put(tepIp, macList);
3174             }
3175         } else {
3176             LOG.debug("RT2 with mac {} l2vni {} from new rd {} and tep ip {}",
3177                     mac, l2vni, rd, tepIp);
3178             isFirstMacUpdateFromTep = true;
3179             Map<String, Uint32> macList = new HashMap<>();
3180             macList.put(mac, l2vni);
3181             Map<String, Map<String, Uint32>> tepIpMacMap = new HashMap<>();
3182             tepIpMacMap.put(tepIp, macList);
3183             rt2TepMap.put(rd, tepIpMacMap);
3184         }
3185         return isFirstMacUpdateFromTep;
3186     }
3187
3188     public boolean deleteFromRt2TepMap(String rd, String tepIp, String mac) {
3189         boolean isLastMacUpdateFromTep = false;
3190         LOG.debug("RT2 withdraw with rd {} mac {} tep-ip {} ", rd, mac, tepIp);
3191         if (rt2TepMap.containsKey(rd)) {
3192             if (rt2TepMap.get(rd).containsKey(tepIp)) {
3193                 if (rt2TepMap.get(rd).get(tepIp).containsKey(mac)) {
3194                     LOG.debug("RT2 Withdraw : Removing the mac {} from Map", mac);
3195                     rt2TepMap.get(rd).get(tepIp).remove(mac);
3196                     if (rt2TepMap.get(rd).get(tepIp).isEmpty()) {
3197                         isLastMacUpdateFromTep = true;
3198                         LOG.debug("RT2 Withdraw : Removing the tep-ip {} from Map", tepIp);
3199                         rt2TepMap.get(rd).remove(tepIp);
3200                         if (rt2TepMap.get(rd).isEmpty()) {
3201                             LOG.debug("RT2 Withdraw : Removing the rd {} from Map", rd);
3202                             rt2TepMap.remove(rd);
3203                         }
3204                     }
3205                 }
3206             }
3207         }
3208         return isLastMacUpdateFromTep;
3209     }
3210
3211     public Collection<String> getTepIPs(String rd) {
3212         final Map<String, Map<String, Uint32>> tepIpMap = rt2TepMap.get(rd);
3213         return tepIpMap != null ? tepIpMap.keySet() : Collections.emptyList();
3214     }
3215
3216     public boolean isBgpConnected() {
3217         return (bgpRouter == null) ? false : bgpRouter.isBgpConnected();
3218     }
3219
3220     public long getLastConnectedTS() {
3221         return (bgpRouter == null) ? 0 : bgpRouter.getLastConnectedTS();
3222     }
3223
3224     public long getConnectTS() {
3225         return (bgpRouter == null) ? 0 : bgpRouter.getConnectTS();
3226     }
3227
3228     public long getStartTS() {
3229         return (bgpRouter == null) ? 0 : bgpRouter.getStartTS();
3230     }
3231
3232     public TTransport getTransport() {
3233         return bgpRouter.getTransport();
3234     }
3235
3236     public int getTotalStaledCount() {
3237         return totalStaledCount;
3238     }
3239
3240     public int getTotalCleared() {
3241         return totalCleared;
3242     }
3243
3244     public static List<Neighbors> getNbrList() {
3245         return nbrList;
3246     }
3247
3248     public BgpCounters getBgpCounters() {
3249         return bgpCountersReference.get();
3250     }
3251
3252     private void startBgpCountersTask() {
3253         if (getBgpCounters() == null && bgpCountersReference.compareAndSet(null,
3254                 new BgpCounters(getBgpSdncMipIp(), metricProvider))) {
3255             bgpCountersTask = executor.scheduleAtFixedRate(bgpCountersReference.get(), 0, 120 * 1000,
3256                     TimeUnit.MILLISECONDS);
3257             LOG.info("Bgp Counters task scheduled for every two minutes.");
3258
3259             bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
3260         }
3261     }
3262
3263     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3264             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3265     private void stopBgpCountersTask() {
3266         final BgpCounters bgpCounters = bgpCountersReference.getAndSet(null);
3267         if (bgpCounters != null) {
3268             bgpCountersTask.cancel(true);
3269             bgpCounters.close();
3270         }
3271     }
3272
3273     private void startBgpAlarmsTask() {
3274         if (getBgpAlarms() == null && bgpAlarmsReference.compareAndSet(null, new BgpAlarms(this))) {
3275             bgpAlarmsReference.get().init();
3276             bgpAlarmsTask = executor.scheduleAtFixedRate(bgpAlarmsReference.get(), 0, 60 * 1000, TimeUnit.MILLISECONDS);
3277             LOG.info("Bgp Alarms task scheduled for every minute.");
3278         } else {
3279             LOG.trace("Bgp Alarms task already scheduled for every minute.");
3280         }
3281     }
3282
3283     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3284             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3285     private void stopBgpAlarmsTask() {
3286         final BgpAlarms bgpAlarms = bgpAlarmsReference.getAndSet(null);
3287         if (bgpAlarms != null) {
3288             bgpAlarmsTask.cancel(true);
3289             bgpAlarms.close();
3290         }
3291     }
3292
3293     public BgpAlarms getBgpAlarms() {
3294         return bgpAlarmsReference.get();
3295     }
3296
3297     public void getPeerStatus(String nbrIp, long nbrAsNum) throws
3298             BgpRouterException, TException {
3299         bgpRouter.getPeerStatus(nbrIp, nbrAsNum);
3300     }
3301
3302     private static String appendNextHopToPrefix(String prefix, String nextHop) {
3303         return prefix + ":" + nextHop;
3304     }
3305
3306     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3307             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3308     private static String extractPrefix(String prefixNextHop) {
3309         return prefixNextHop.split(":")[0];
3310     }
3311
3312     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3313             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3314     private static String extractNextHop(String prefixNextHop) {
3315         return prefixNextHop.split(":")[1];
3316     }
3317
3318     private static String extractMd5Secret(final Neighbors val) {
3319         String md5Secret = null;
3320         TcpSecurityOption tcpSecOpt = val.getTcpSecurityOption();
3321         if (tcpSecOpt != null) {
3322             if (tcpSecOpt instanceof TcpMd5SignatureOption) {
3323                 md5Secret = ((TcpMd5SignatureOption) tcpSecOpt).getTcpMd5SignaturePassword().getValue();
3324             } else { // unknown TcpSecurityOption
3325                 LOG.debug("neighbors  Ignored unknown tcp-security-option of peer {}", val.getAddress().getValue());
3326             }
3327         }
3328         return md5Secret;
3329     } // private method extractMd5Secret
3330
3331     @SuppressWarnings("checkstyle:IllegalCatch")
3332     @Override
3333     public ListenableFuture<RpcResult<InitiateEorOutput>> initiateEor(InitiateEorInput input) {
3334         boolean returnError = false;
3335         String msg = null;
3336         String neighborIp = null;
3337         if (!isBGPEntityOwner()) {
3338             msg = String.format("RPC triggered in Non-EoS Owner");
3339             return Futures.immediateFuture(
3340                     RpcResultBuilder.<InitiateEorOutput>failed().withError(RpcError.ErrorType.APPLICATION,
3341                             msg).build());
3342         }
3343         if (input == null) {
3344             msg = String.format("BGP invalid input for EoR");
3345             LOG.error("Error : {}", msg);
3346             returnError = true;
3347         } else {
3348             neighborIp = input.getNeighborIp();
3349         }
3350         if (eorSupressedDuetoUpgradeFlag.get() == false) {
3351             msg = String.format("EoR triggerd by RBU-RPC call before replay"
3352                     + "of BGP configuration (or) BGP not restarted");
3353             LOG.error("Error : {}", msg);
3354         }
3355         if ("ALL".compareToIgnoreCase(neighborIp) == 0) {
3356             //send EoR for all the neighbor
3357             LOG.error("EoR trigger received to ALL neighbors");
3358             final int numberOfEORRetries = 3;
3359             RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
3360             do {
3361                 try {
3362                     BgpRouter br = bgpRouter;
3363                     br.sendEOR();
3364                     LOG.debug("RPC: sendEOR {} successful", br);
3365                     break;
3366                 } catch (Exception e) {
3367                     eorRetry.errorOccured();
3368                     LOG.error("Replay:sedEOR() received exception:", e);
3369                 }
3370             } while (eorRetry.shouldRetry());
3371             eorSupressedDuetoUpgradeFlag.set(false);
3372         } else if (InetAddresses.isInetAddress(neighborIp)) {
3373             //send EoR for only one neighbor
3374             msg = String.format("Inidividual neighbors EoR is not supported");
3375             LOG.warn("Error : {}", msg);
3376             returnError = true;
3377         } else {
3378             //error
3379             msg = String.format("RPC: initiateEor: Invalid input ");
3380             LOG.warn("Error : {}", msg);
3381             returnError = true;
3382         }
3383         if (returnError) {
3384             return Futures.immediateFuture(
3385                     RpcResultBuilder.<InitiateEorOutput>failed().withError(RpcError.ErrorType.APPLICATION,
3386                             msg).build());
3387         }
3388         InitiateEorOutput initiateEorOutput =
3389                 new InitiateEorOutputBuilder().setRetVal(0L).build();
3390         return Futures.immediateFuture(RpcResultBuilder.<InitiateEorOutput>success()
3391                 .withResult(initiateEorOutput).build());
3392     }
3393 }