Timer and log enhancements in BGP
[netvirt.git] / bgpmanager / impl / src / main / java / org / opendaylight / netvirt / bgpmanager / BgpConfigurationManager.java
index de83f6ab613c697764c8bd24f38dfa4d961c68c7..a84eec3ed6bb006a7cd2e7df3facc2bd8f948a4a 100755 (executable)
@@ -7,12 +7,14 @@
  */
 package org.opendaylight.netvirt.bgpmanager;
 
+import static org.opendaylight.netvirt.bgpmanager.oam.BgpConstants.HISTORY_LIMIT;
+import static org.opendaylight.netvirt.bgpmanager.oam.BgpConstants.HISTORY_THRESHOLD;
+
 import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
 import com.google.common.net.InetAddresses;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import io.netty.util.concurrent.GlobalEventExecutor;
-import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.net.InetAddress;
@@ -32,20 +34,22 @@ import java.util.Timer;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
-import javax.annotation.Nullable;
 import javax.annotation.PreDestroy;
 import javax.inject.Inject;
 import javax.inject.Singleton;
 import org.apache.thrift.TApplicationException;
 import org.apache.thrift.TException;
 import org.apache.thrift.transport.TTransport;
+import org.eclipse.jdt.annotation.Nullable;
 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
@@ -79,6 +83,10 @@ import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
+import org.opendaylight.ovsdb.utils.mdsal.utils.TransactionHistory;
+import org.opendaylight.ovsdb.utils.mdsal.utils.TransactionType;
+import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebfd.rev190219.BfdConfig;
+import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebfd.rev190219.BfdConfigBuilder;
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.AddressFamily;
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
@@ -88,6 +96,7 @@ import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev1509
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
+import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.DcgwTepList;
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
@@ -107,6 +116,9 @@ import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev1509
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Vrfs;
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsBuilder;
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsKey;
+import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTep;
+import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTepBuilder;
+import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTepKey;
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamilies;
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesBuilder;
 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighbors.AddressFamiliesKey;
@@ -149,7 +161,15 @@ public class BgpConfigurationManager {
     private static final String DEF_CHOST = "255.255.255.255"; // Invalid Host IP
     private static final String DEF_CPORT = "7644";
     private static final String DEF_BGP_SDNC_MIP = "127.0.0.1";
-    private static final String BGP_SDNC_MIP = "vpnservice.bgp.thrift.sdnc.mip";
+    //vpnservice.bgp.thrift.bgp.mip is the MIP present with ODL. Here we open 6644 port
+    private static final String BGP_SDNC_MIP = "vpnservice.bgp.thrift.bgp.mip";
+    private static final String BGP_GR_RESTART_TIMER_PROPERTY = "vpnservice.bgp.gr.timer";
+    private static final String BGP_KA_TIMER_PROPERTY = "vpnservice.bgp.ka.timer";
+    private static final String BGP_HOLD_TIMER_PROPERTY = "vpnservice.bgp.hold.timer";
+    private static final String BGP_EOR_DELAY_PROPERTY = "vpnservice.bgp.eordelay";
+    private static final int DEF_BGP_KA_TIME = 60;
+    private static final int DEF_BGP_HOLD_TIME = 180;
+    private static final int DEF_BGP_GR_TIME = 4000;
     private static final int RESTART_DEFAULT_GR = 90;
     private static final int DS_RETRY_COUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
@@ -161,6 +181,10 @@ public class BgpConfigurationManager {
     private static final String DEL_WARN = "Config store updated; undo with Add if needed.";
     private static final String UPD_WARN = "Update operation not supported; Config store updated;"
             + " restore with another Update if needed.";
+    private static long bgp_as_num = 0;
+    private int bgpKaTime = 0;
+    private int bgpHoldTime = 0;
+    private int bgpGrRestartTime = 0;
 
     private static final Class<?>[] REACTORS = {
         ConfigServerReactor.class, AsIdReactor.class,
@@ -168,7 +192,7 @@ public class BgpConfigurationManager {
         NeighborsReactor.class, UpdateSourceReactor.class,
         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
         NetworksReactor.class, VrfsReactor.class, BgpReactor.class,
-        MultipathReactor.class, VrfMaxpathReactor.class
+        MultipathReactor.class, VrfMaxpathReactor.class, BfdConfigReactor.class
     };
 
     private IBgpManager bgpManager;
@@ -228,6 +252,7 @@ public class BgpConfigurationManager {
     private final EntityOwnershipCandidateRegistration candidateRegistration;
     private final EntityOwnershipListenerRegistration entityListenerRegistration;
     private final MetricProvider metricProvider;
+    private final TransactionHistory bgpUpdatesHistory;
 
     @Inject
     public BgpConfigurationManager(final DataBroker dataBroker,
@@ -243,21 +268,35 @@ public class BgpConfigurationManager {
         this.bundleContext = bundleContext;
         this.bgpUtil = bgpUtil;
         this.metricProvider = metricProvider;
-        String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
         hostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
         portStartup = getProperty(CONFIG_PORT, DEF_CPORT);
+        bgpKaTime =
+                Integer.parseInt(getProperty(BGP_KA_TIMER_PROPERTY,
+                        Integer.toString(DEF_BGP_KA_TIME)));
+        bgpHoldTime =
+                Integer.parseInt(getProperty(BGP_HOLD_TIMER_PROPERTY,
+                        Integer.toString(DEF_BGP_HOLD_TIME)));
+        bgpGrRestartTime =
+                Integer.parseInt(getProperty(BGP_GR_RESTART_TIMER_PROPERTY,
+                        Integer.toString(DEF_BGP_GR_TIME)));
         LOG.info("ConfigServer at {}:{}", hostStartup, portStartup);
         VtyshCli.setHostAddr(hostStartup);
         ClearBgpCli.setHostAddr(hostStartup);
-        bgpRouter = BgpRouter.newInstance(this::getConfig, this::isBGPEntityOwner);
+        bgpUpdatesHistory = new TransactionHistory(HISTORY_LIMIT, HISTORY_THRESHOLD);
+        bgpRouter = BgpRouter.newInstance(this::getConfig, this::isBGPEntityOwner, bgpUpdatesHistory);
         delayEorSeconds = Integer.parseInt(getProperty(BGP_EOR_DELAY, DEF_BGP_EOR_DELAY));
-        registerCallbacks();
 
         entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 
         candidateRegistration = registerEntityCandidate(entityOwnershipService);
         entityListenerRegistration = registerEntityListener(entityOwnershipService);
 
+        /*register callbacks for reactors, shall be called after EoS registration.
+         * as listeners user EoS service to identify Owner node. Listener call-backs
+         * can get triggered immediately after registeration (before EoS register complete)
+        */
+        registerCallbacks();
+
         LOG.info("BGP Configuration manager initialized");
         initer.countDown();
 
@@ -276,6 +315,7 @@ public class BgpConfigurationManager {
                 }
             }
 
+            String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
             if (InetAddresses.isInetAddress(getBgpSdncMipIp())) {
                 InetSocketAddress bgpThriftServerSocketAddr = new InetSocketAddress(getBgpSdncMipIp(),
                         Integer.parseInt(updatePort));
@@ -313,6 +353,10 @@ public class BgpConfigurationManager {
         this.cfgReplayEndTime = cfgReplayEndTime;
     }
 
+    public TransactionHistory getBgpUpdatesHistory() {
+        return bgpUpdatesHistory;
+    }
+
     public long getCfgReplayStartTime() {
         return cfgReplayStartTime;
     }
@@ -438,6 +482,10 @@ public class BgpConfigurationManager {
     }
 
     public boolean isBGPEntityOwner() {
+        if (entityOwnershipUtils == null) {
+            LOG.error("entityOwnershipUtils is NULL when listener callbacks fired");
+            return false;
+        }
         return entityOwnershipUtils.isEntityOwner(new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME), 0, 1);
     }
 
@@ -500,6 +548,17 @@ public class BgpConfigurationManager {
             bgpRouter.configServerUpdated();
 
             synchronized (BgpConfigurationManager.this) {
+                if (bgp_as_num != 0) {
+                    try {
+                        bgpRouter.stopBgp(bgp_as_num);
+                        stopBgpCountersTask();
+                        stopBgpAlarmsTask();
+                    } catch (TException | BgpRouterException e) {
+                        LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
+                    }
+                } else {
+                    LOG.debug("bgp as-id is null while removing config-server");
+                }
                 bgpRouter.disconnect();
             }
         }
@@ -540,6 +599,7 @@ public class BgpConfigurationManager {
                 return;
             }
             LOG.debug("received add router config asNum {}", val.getLocalAs());
+            bgp_as_num = val.getLocalAs().longValue();
             synchronized (BgpConfigurationManager.this) {
                 BgpRouter br = getClient(YANG_OBJ);
                 if (br == null) {
@@ -573,6 +633,7 @@ public class BgpConfigurationManager {
             synchronized (BgpConfigurationManager.this) {
                 long asNum = val.getLocalAs();
                 BgpRouter br = getClient(YANG_OBJ);
+                bgp_as_num = 0;
                 if (br == null) {
                     LOG.debug("{} Unable to process remove for asNum {}; {} {}", YANG_OBJ, asNum,
                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
@@ -855,6 +916,13 @@ public class BgpConfigurationManager {
                 if (bgpAlarms != null) {
                     bgpAlarms.clearBgpNbrDownAlarm(peerIp);
                 }
+
+                if (bgpUtil.isBfdEnabled()) {
+                    final BgpCounters bgpCounters = getBgpCounters();
+                    if (bgpCounters != null) {
+                        bgpCounters.clearBfdNbrCounters(peerIp);
+                    }
+                }
             }
         }
 
@@ -1392,18 +1460,6 @@ public class BgpConfigurationManager {
         }
     }
 
-    protected void activateMIP() {
-        try {
-            LOG.trace("BgpReactor: Executing MIP Activate command");
-            Runtime.getRuntime().exec("cluster ip -a sdnc_bgp_mip");
-            Runtime.getRuntime().exec("cluster ip -a sdnc_os_mip");
-            LOG.trace("bgpMIP Activated");
-
-        } catch (IOException io) {
-            LOG.error("IO Exception got while activating mip: {}", io.getMessage());
-        }
-    }
-
     public class BgpReactor
             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
             implements ClusteredDataTreeChangeListener<Bgp> {
@@ -1429,7 +1485,6 @@ public class BgpConfigurationManager {
                 if (!isBGPEntityOwner()) {
                     return;
                 }
-                activateMIP();
             }
         }
 
@@ -1593,7 +1648,19 @@ public class BgpConfigurationManager {
 
         @Override
         protected void remove(InstanceIdentifier<VrfMaxpath> iid, VrfMaxpath vrfMaxPathVal) {
-            executor.execute(new VrfMaxPathConfigurator(vrfMaxPathVal));
+            if (isBGPEntityOwner()) {
+                synchronized (BgpConfigurationManager.this) {
+                    BgpRouter br = getClient(YANG_OBJ);
+                    if (br != null) {
+                        try {
+                            br.multipaths(vrfMaxPathVal.getRd(), BgpConstants.BGP_DEFAULT_MULTIPATH);
+                            LOG.debug("Del Maxpath for vrf: {} ", vrfMaxPathVal.getRd());
+                        } catch (TException | BgpRouterException e) {
+                            LOG.error(YANG_OBJ + " del received exception:", e);
+                        }
+                    }
+                }
+            }
         }
 
         @Override
@@ -1615,6 +1682,96 @@ public class BgpConfigurationManager {
         }
     }
 
+    public class BfdConfigReactor
+            extends AsyncDataTreeChangeListenerBase<BfdConfig, BfdConfigReactor>
+            implements ClusteredDataTreeChangeListener<BfdConfig> {
+
+        private static final String YANG_OBJ = "BfdConfig ";
+
+        public BfdConfigReactor() {
+            super(BfdConfig.class, BfdConfigReactor.class);
+        }
+
+        @Override
+        protected void add(InstanceIdentifier<BfdConfig> iid, BfdConfig val) {
+            if (!isBGPEntityOwner()) {
+                return;
+            }
+            BgpRouter br = getClient(YANG_OBJ);
+            LOG.debug("received bfd config: bfd enabled {} min-rx {} min-tx {} detect-mul {} mhop {}",
+                    val.isBfdEnabled(), val.getMinRx(), val.getMinTx(),
+                    val.getDetectMult(), val.isMultihop());
+            if (br == null) {
+                LOG.debug(YANG_OBJ + "{} Unable to process add  {}",
+                        BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
+                return;
+            }
+            if (val.isBfdEnabled() == false) {
+                LOG.debug("BFD not enabled. Ignoring the config add");
+                return;
+            }
+            int minRx = val.getMinRx().intValue();
+            int minTx = val.getMinTx().intValue();
+            int detectMult = val.getDetectMult().intValue();
+            boolean multiHop = val.isMultihop();
+            try {
+                br.addBfd(detectMult, minRx, minTx,multiHop);
+            } catch (TException | BgpRouterException e) {
+                LOG.error("{} get {}, Add received exception;", YANG_OBJ, ADD_WARN, e);
+            }
+        }
+
+        @Override
+        protected BfdConfigReactor getDataTreeChangeListener() {
+            return BfdConfigReactor.this;
+        }
+
+        @Override
+        protected InstanceIdentifier<BfdConfig> getWildCardPath() {
+            return InstanceIdentifier.create(BfdConfig.class);
+        }
+
+        @Override
+        protected void remove(InstanceIdentifier<BfdConfig> iid, BfdConfig val) {
+            if (!isBGPEntityOwner()) {
+                return;
+            }
+            LOG.debug("received bfd config removal");
+            BgpRouter br = getClient(YANG_OBJ);
+            if (br == null) {
+                LOG.debug("{} Unable to process del {}  {}", YANG_OBJ,
+                        BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
+                return;
+            }
+            try {
+                br.delBfd();
+            } catch (TException | BgpRouterException e) {
+                LOG.error("{} get {}, Del received exception;", YANG_OBJ, ADD_WARN, e);
+            }
+
+        }
+
+        @Override
+        protected void update(InstanceIdentifier<BfdConfig> iid,
+                              BfdConfig oldval, BfdConfig newval) {
+            LOG.debug("received bfd config: updated oldval bfd enabled {}"
+                    + "min-rx {} min-tx {} detect-mul {} mhop {}",
+                    oldval.isBfdEnabled(), oldval.getMinRx(), oldval.getMinTx(),
+                    oldval.getDetectMult(), oldval.isMultihop());
+            LOG.debug("received bfd config: updated newval bfd enabled {}"
+                    + "min-rx {} min-tx {} detect-mul {} mhop {}",
+                    newval.isBfdEnabled(), newval.getMinRx(), newval.getMinTx(),
+                    newval.getDetectMult(), newval.isMultihop());
+            if (oldval.isBfdEnabled()) {
+                LOG.debug("deleting bfd config on an update");
+                remove(iid, oldval);
+            }
+            LOG.debug("adding bfd config on an update");
+            add(iid, newval);
+        }
+    }
+
+
     public boolean isIpAvailable(String odlip) {
 
         try {
@@ -1677,39 +1834,45 @@ public class BgpConfigurationManager {
             cancelPreviousReplayJob();
         }
         Runnable task = () -> {
-            LOG.info("running bgp replay task ");
-            if (get() == null) {
-                String host = getConfigHost();
-                int port = getConfigPort();
-                LOG.info("connecting  to bgp host {} ", host);
-                bgpRouter.connect(host, port);
-                LOG.info("no config to push in bgp replay task ");
-                return;
-            }
-            setStaleStartTime(System.currentTimeMillis());
-            LOG.info("started creating stale fibDSWriter  map ");
-            createStaleFibMap();
-            setStaleEndTime(System.currentTimeMillis());
-            LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
-            LOG.info("started bgp config replay ");
-            setCfgReplayStartTime(System.currentTimeMillis());
-            boolean replaySucceded = replay();
-            setCfgReplayEndTime(System.currentTimeMillis());
-            LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
-            if (replaySucceded) {
-                LOG.info("starting the stale cleanup timer");
-                long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
-                setStaleCleanupTime(routeSyncTime);
-                routeCleanupFuture = executor.schedule(new RouteCleanup(), routeSyncTime, TimeUnit.SECONDS);
-            } else {
-                staledFibEntriesMap.clear();
+            try {
+                LOG.info("running bgp replay task ");
+                if (get() == null) {
+                    String host = getConfigHost();
+                    int port = getConfigPort();
+                    LOG.info("connecting  to bgp host {} ", host);
+                    bgpRouter.connect(host, port);
+                    LOG.info("no config to push in bgp replay task ");
+                    return;
+                }
+                setStaleStartTime(System.currentTimeMillis());
+                LOG.info("started creating stale fibDSWriter  map ");
+                createStaleFibMap();
+                setStaleEndTime(System.currentTimeMillis());
+                LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
+                LOG.info("started bgp config replay ");
+                setCfgReplayStartTime(System.currentTimeMillis());
+                boolean replaySucceded = replay();
+                setCfgReplayEndTime(System.currentTimeMillis());
+                LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
+                if (replaySucceded) {
+                    long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
+                    setStaleCleanupTime(routeSyncTime);
+                    LOG.error("starting the stale cleanup timer: {} seconds", routeSyncTime);
+                    routeCleanupFuture = executor.schedule(new RouteCleanup(), routeSyncTime, TimeUnit.SECONDS);
+                } else {
+                    LOG.error("skipping stale cleanup, may be due to exception while replay");
+                    staledFibEntriesMap.clear();
+                }
+            } catch (InterruptedException | TimeoutException | ExecutionException eCancel) {
+                LOG.error("Stale Cleanup Task Cancelled", eCancel);
             }
         };
         lastReplayJobFt = executor.submit(task);
     }
 
     private boolean previousReplayJobInProgress() {
-        return lastReplayJobFt != null && !lastReplayJobFt.isDone();
+        return ((lastReplayJobFt != null && !lastReplayJobFt.isDone())
+                || (routeCleanupFuture != null && !routeCleanupFuture.isDone()));
     }
 
     private void cancelPreviousReplayJob() {
@@ -1819,6 +1982,9 @@ public class BgpConfigurationManager {
 
     public void onUpdatePushRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
                                   String macaddress, int label, int l2label, String routermac, af_afi afi) {
+        PrefixUpdateEvent prefixUpdateEvent = new PrefixUpdateEvent(protocolType,rd,prefix,plen,nextHop,
+                macaddress,label,l2label,routermac,afi);
+        bgpUpdatesHistory.addToHistory(TransactionType.ADD, prefixUpdateEvent);
         boolean addroute = false;
         boolean macupdate = false;
         long l3vni = 0L;
@@ -1889,6 +2055,9 @@ public class BgpConfigurationManager {
 
     public void onUpdateWithdrawRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
             String macaddress) {
+        PrefixWithdrawEvent prefixWithdrawEvent = new PrefixWithdrawEvent(protocolType,rd,prefix,plen,
+                nextHop,macaddress);
+        bgpUpdatesHistory.addToHistory(TransactionType.ADD, prefixWithdrawEvent);
         long vni = 0L;
         boolean macupdate = false;
         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
@@ -1929,6 +2098,28 @@ public class BgpConfigurationManager {
         }
     }
 
+    public void peerDown(String ipAddress, long asNumber) {
+        List<String> tepIpList = bgpUtil.getDcgwTepConfig(ipAddress);
+        if (tepIpList == null) {
+            LOG.error("No Tep IP configured for DCGW {} on a peerDown", ipAddress);
+            return;
+        }
+        tepIpList.forEach(tepIp -> {
+            bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW);
+        });
+    }
+
+    public void peerUp(String ipAddress, long asNumber) {
+        List<String> tepIpList = bgpUtil.getDcgwTepConfig(ipAddress);
+        if (tepIpList == null) {
+            LOG.error("No Tep IP configured for DCGW {} on a peerUp", ipAddress);
+            return;
+        }
+        tepIpList.forEach(tepIp -> {
+            bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW);
+        });
+    }
+
     private static boolean isRouteModified(int label, Long labelInStaleMap) {
         return labelInStaleMap != null && !labelInStaleMap.equals(Long.valueOf(label));
     }
@@ -1981,53 +2172,46 @@ public class BgpConfigurationManager {
                     final String md5password = extractMd5Secret(replayNbr.getNbr());
                     br.addNeighbor(replayNbr.getNbr().getAddress().getValue(),
                             replayNbr.getNbr().getRemoteAs().longValue(), md5password);
-                    UpdateSource us = replayNbr.getNbr().getUpdateSource();
-                    if (us != null) {
-                        LOG.debug("Replaying updatesource along with nbr: {} US-ip: {} to peer {}",
-                                replayNbr.getNbr().getAddress().getValue(),
-                                us.getSourceIp().getValue(),
-                                us.getPeerIp().getValue());
-                        br.addUpdateSource(us.getPeerIp().getValue(),
-                                us.getSourceIp().getValue());
-                    }
                     replayDone = true;
+                } catch (TApplicationException tae) {
+                    LOG.debug("Replaying addNbr {}, tapplicationexception: ",
+                            replayNbr.getNbr().getAddress().getValue(), tae);
+                    if (tae.getType() == BgpRouterException.BGP_ERR_PEER_EXISTS) {
+                        LOG.debug("Replaying addNbr Neighbor already present");
+                        replayDone = true;
+                    } else {
+                        LOG.error("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), tae);
+                    }
                 } catch (TException | BgpRouterException eNbr) {
                     LOG.debug("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), eNbr);
                 }
-                boolean replaySuccess = true;
-                replaySuccess = replaySuccess && replayDone;
+
                 LOG.debug("Replay addNbr {} successful", replayNbr.getNbr().getAddress().getValue());
 
                 //Update Source handling
                 UpdateSource us = replayNbr.getNbr().getUpdateSource();
-                if (replayDone == false && us != null) {
+                if (us != null) {
                     LOG.debug("Replaying updatesource {} to peer {}", us.getSourceIp().getValue(),
                             us.getPeerIp().getValue());
-                    replayDone = false;
                     try {
                         br.addUpdateSource(us.getPeerIp().getValue(),
                                 us.getSourceIp().getValue());
-                        replayDone = true;
                     } catch (TException | BgpRouterException eUs) {
                         LOG.debug("Replaying UpdateSource for Nbr {}, exception:",
                                 replayNbr.getNbr().getAddress().getValue(), eUs);
                     }
                     LOG.debug("Replay updatesource {} successful", us.getSourceIp().getValue());
-                    replaySuccess = replaySuccess && replayDone;
                 }
                 //Ebgp Multihope
                 EbgpMultihop en = replayNbr.getNbr().getEbgpMultihop();
                 if (en != null) {
-                    replayDone = false;
                     try {
                         br.addEbgpMultihop(en.getPeerIp().getValue(),
                                 en.getNhops().intValue());
-                        replayDone = true;
                     } catch (TException | BgpRouterException eEbgpMhop) {
                         LOG.debug("Replaying EbgpMultihop for Nbr {}, exception: ",
                                 replayNbr.getNbr().getAddress().getValue(), eEbgpMhop);
                     }
-                    replaySuccess = replaySuccess && replayDone;
                 }
 
                 //afs
@@ -2036,24 +2220,25 @@ public class BgpConfigurationManager {
                     for (AddressFamilies af : afs) {
                         af_afi afi = af_afi.findByValue(af.getAfi().intValue());
                         af_safi safi = af_safi.findByValue(af.getSafi().intValue());
-                        replayDone = false;
                         try {
                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
-                            replayDone = true;
                         } catch (TException | BgpRouterException eAFs) {
                             LOG.debug("Replaying AddressFamily for Nbr {}, exception:",
                                     replayNbr.getNbr().getAddress().getValue(), eAFs);
                         }
-                        replaySuccess = replaySuccess && replayDone;
                     }
                 }
                 //replay is success --> no need to replay this nbr in next iteration.
-                replayNbr.setShouldRetry(replaySuccess ? false : true);
+                replayNbr.setShouldRetry(replayDone ? false : true);
             }
         } while (nbrRetry.decrementAndRetry());
         boolean replaySuccess = true;
         for (ReplayNbr replayNbr : replayNbrList) {
             replaySuccess = replaySuccess && !replayNbr.isShouldRetry();
+            if (replaySuccess == false) {
+                LOG.error("replayNbrConfig: will be cancelling stale cleanup, cfg nbr: {} Failed:",
+                        replayNbr.getNbr().getAddress().getValue());
+            }
         }
         return replaySuccess;
     }
@@ -2097,15 +2282,15 @@ public class BgpConfigurationManager {
     }
 
     @SuppressWarnings("checkstyle:IllegalCatch")
-    public synchronized boolean replay() {
+    public synchronized boolean replay() throws InterruptedException, TimeoutException, ExecutionException {
         boolean replaySucceded = true;
+        boolean doRouteSync = false;
         String host = getConfigHost();
         int port = getConfigPort();
         LOG.error("connecting  to bgp host {} ", host);
         boolean res = bgpRouter.connect(host, port);
         if (!res) {
-            LOG.error("Cannot connect to BGP config server at {}:{}{}", host, port,
-                    config != null ? "; Configuration Replay aborted" : "");
+            LOG.error("Cannot connect to BGP config server at {} {}", host, port);
             return replaySucceded;
         }
         config = getConfig();
@@ -2122,7 +2307,7 @@ public class BgpConfigurationManager {
         long asNum = asId.getLocalAs();
         IpAddress routerId = asId.getRouterId();
         String rid = routerId == null ? "" : routerId.stringValue();
-        int stalepathTime = (int) getStalePathtime(RESTART_DEFAULT_GR, config.getAsId());
+        int stalepathTime = (int) getStalePathtime(bgpGrRestartTime, config.getAsId());
         boolean announceFbit = true;
         boolean replayDone = false;
         final int numberOfStartBgpRetries = 3;
@@ -2130,7 +2315,7 @@ public class BgpConfigurationManager {
         do {
             try {
                 LOG.debug("Replaying BGPConfig ");
-                br.startBgp(asNum, rid, stalepathTime, announceFbit);
+                br.startBgp(asNum, rid, bgpKaTime, bgpHoldTime, stalepathTime, announceFbit);
                 LOG.debug("Replay BGPConfig successful");
                 replayDone = true;
                 break;
@@ -2139,8 +2324,8 @@ public class BgpConfigurationManager {
                     LOG.debug("Starting the routesync for exception", bre);
                     startBgpRetry.errorOccured();
                     if (!startBgpRetry.shouldRetry()) {
-                        LOG.debug("starting route sync for BgpRouter exception");
-                        doRouteSync();
+                        doRouteSync = true;
+                        replayDone = true;
                     }
                 } else {
                     LOG.error("Replay: startBgp() received exception error {} : ",
@@ -2152,15 +2337,15 @@ public class BgpConfigurationManager {
                     LOG.debug("Starting the routesync for exception", tae);
                     startBgpRetry.errorOccured();
                     if (!startBgpRetry.shouldRetry()) {
-                        LOG.debug("starting route sync for Thrift BGP_ERR_ACTIVE exception");
-                        doRouteSync();
+                        doRouteSync = true;
+                        replayDone = true;
                     }
                 } else if (tae.getType() == BgpRouterException.BGP_ERR_COMMON_FAILURE) {
                     LOG.debug("Starting the routesync for AS-ID started exception", tae);
                     startBgpRetry.errorOccured();
                     if (!startBgpRetry.shouldRetry()) {
-                        LOG.debug("starting route sync for Thrift BGP_ERR_COMMON_FAILURE exception");
-                        doRouteSync();
+                        doRouteSync = true;
+                        replayDone = true;
                     }
                 } else {
                     LOG.error("Replay: startBgp() received exception type {}: ",
@@ -2183,10 +2368,31 @@ public class BgpConfigurationManager {
          * commenting this due to a bug with QBGP. Will uncomment once QBGP fix is done.
          * This wont have any functional impacts
          */
-        try {
-            br.delayEOR(delayEorSeconds);
-        } catch (TException | BgpRouterException e) {
-            LOG.error("Replay: delayEOR() number of seconds to wait for EOR from ODL:", e);
+        //try {
+        //    br.delayEOR(delayEorSeconds);
+        //} catch (TException | BgpRouterException e) {
+        //    LOG.error("Replay: delayEOR() number of seconds to wait for EOR from ODL:", e);
+        //}
+
+        BfdConfig bfdConfig = bgpUtil.getBfdConfig();
+        if (bfdConfig != null) {
+            if (bfdConfig.isBfdEnabled()) {
+                LOG.debug("Replaying bfd config min-rx {} min-tx {} detect-mul {} mhop {}",
+                        bfdConfig.getMinRx(), bfdConfig.getMinTx(),
+                        bfdConfig.getDetectMult(), bfdConfig.isMultihop());
+                try {
+                    br.addBfd(bfdConfig.getDetectMult().intValue(), bfdConfig.getMinRx().intValue(),
+                            bfdConfig.getMinTx().intValue(), bfdConfig.isMultihop());
+                } catch (TApplicationException tae) {
+                    if (tae.getType() == BgpRouterException.BGP_ERR_PEER_EXISTS) {
+                        LOG.debug("Replay:addBfd() received exception", tae);
+                    } else {
+                        LOG.error("Replay:addBfd() received exception", tae);
+                    }
+                } catch (TException | BgpRouterException e) {
+                    LOG.error("Replay:addBfd() received exception", e);
+                }
+            }
         }
 
         List<Neighbors> neighbors = config.getNeighbors();
@@ -2210,14 +2416,13 @@ public class BgpConfigurationManager {
         }
 
         GracefulRestart gracefulRestart = config.getGracefulRestart();
-        if (gracefulRestart != null) {
-            try {
-                br.addGracefulRestart(gracefulRestart.getStalepathTime().intValue());
-            } catch (TException | BgpRouterException e) {
-                LOG.error("Replay:addGr() received exception", e);
-            }
+        bgpGrRestartTime = ((gracefulRestart != null)
+                ? gracefulRestart.getStalepathTime().intValue() : bgpGrRestartTime);
+        try {
+            br.addGracefulRestart(bgpGrRestartTime);
+        } catch (Exception e) {
+            LOG.error("Replay:addGr() received exception: ", e);
         }
-
         List<Vrfs> vrfs = config.getVrfs();
         if (vrfs == null) {
             vrfs = new ArrayList<>();
@@ -2299,13 +2504,11 @@ public class BgpConfigurationManager {
 
         //send End of Rib Marker to Qthriftd.
         final int numberOfEORRetries = 3;
-        replayDone = false;
         RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
         do {
             try {
                 br.sendEOR();
-                LOG.debug("Replay sendEOR {} successful");
-                replayDone = true;
+                LOG.debug("Replay sendEOR() successful");
                 break;
             } catch (Exception e) {
                 eorRetry.errorOccured();
@@ -2313,7 +2516,13 @@ public class BgpConfigurationManager {
             }
         } while (eorRetry.shouldRetry());
 
-        return replaySucceded && replayDone;
+        if (doRouteSync) {
+            LOG.debug("starting route sync for Thrift BGP_ERR_COMMON_FAILURE exception "
+                      + "happened earlier");
+            doRouteSync();
+        }
+
+        return replaySucceded;
     }
 
     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
@@ -2347,6 +2556,34 @@ public class BgpConfigurationManager {
         update(iid, dto);
     }
 
+    public void startBfd(long detectMult, long minRx, long minTx, boolean multiHop) {
+        InstanceIdentifier.InstanceIdentifierBuilder<BfdConfig> iib =
+                InstanceIdentifier.builder(BfdConfig.class);
+        InstanceIdentifier<BfdConfig> iid = iib.build();
+        BfdConfig dto = new BfdConfigBuilder()
+                .setBfdEnabled(true)
+                .setMultihop(multiHop)
+                .setMinRx(minRx)
+                .setMinTx(minTx)
+                .setDetectMult(detectMult)
+                .build();
+        update(iid, dto);
+    }
+
+    public void addDcgwTep(String dcgwIp, String tepIp) {
+        InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
+                InstanceIdentifier.builder(Bgp.class)
+                        .child(DcgwTepList.class)
+                        .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
+        InstanceIdentifier<DcgwTep> iid = iib.build();
+        ArrayList<String> tepList = new ArrayList<String>();
+        tepList.add(tepIp);
+        DcgwTep dto = new DcgwTepBuilder().setDcGwIp(dcgwIp).setTepIps(tepList)
+                .build();
+        update(iid, dto);
+        bgpUtil.removeOrUpdateLBGroups(tepIp,NwConstants.MOD_FLOW);
+    }
+
     public void addLogging(String fileName, String logLevel) {
         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
@@ -2482,6 +2719,9 @@ public class BgpConfigurationManager {
             LOG.error("Error adding VRF to datastore", e);
             throw new RuntimeException(e);
         }
+
+        // enable multipath by default in all VRFs
+        setMultipaths(rd, BgpConstants.BGP_DEFAULT_MULTIPATH);
     }
 
     public void stopConfig() {
@@ -2498,6 +2738,50 @@ public class BgpConfigurationManager {
         delete(iid);
     }
 
+    public void stopBfd() {
+        InstanceIdentifier.InstanceIdentifierBuilder<BfdConfig> iib =
+                InstanceIdentifier.builder(BfdConfig.class);
+        InstanceIdentifier<BfdConfig> iid = iib.build();
+        delete(iid);
+    }
+
+    public void delDcgwTep(String dcgwIp, String tepIp) {
+        if (tepIp == null) {
+            InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
+                    InstanceIdentifier.builder(Bgp.class)
+                            .child(DcgwTepList.class)
+                            .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
+            InstanceIdentifier<DcgwTep> iid = iib.build();
+            delete(iid);
+        } else {
+            InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
+                    InstanceIdentifier.builder(Bgp.class)
+                            .child(DcgwTepList.class)
+                            .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
+            InstanceIdentifier<DcgwTep> iid = iib.build();
+            List<String> tepIpList = bgpUtil.getDcgwTepConfig(dcgwIp);
+            if (tepIpList == null) {
+                LOG.error("No Tep IP configured for DCGW {} on deleting the dcgwtep", dcgwIp);
+                return;
+            }
+            List<String> newTepIpList = new ArrayList<String>();
+            tepIpList.forEach(tep -> {
+                if (!tep.equals(tepIp)) {
+                    newTepIpList.add(tep);
+                }
+            });
+            DcgwTep dto = new DcgwTepBuilder().setDcGwIp(dcgwIp).setTepIps(newTepIpList)
+                    .build();
+            try {
+                SingleTransactionDataBroker.syncWrite(dataBroker,
+                        LogicalDatastoreType.CONFIGURATION, iid, dto);
+            } catch (TransactionCommitFailedException e) {
+                LOG.error("delDcgwTep: Error deleting DCGW Tep", e);
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
     public void delLogging() {
         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
@@ -2565,6 +2849,7 @@ public class BgpConfigurationManager {
             LOG.error("delVrf: vrf {}, addressFamily invalid", rd);
             return false;
         }
+        delMultipaths(rd);
         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
         if (addressFamily.equals(AddressFamily.IPV4)) {
             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
@@ -2576,6 +2861,9 @@ public class BgpConfigurationManager {
             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
         }
+        LOG.debug("delVrf: Received Delete VRF : rd:{}, address family: {} {}", rd,
+                adfBuilder.getAfi(), adfBuilder.getSafi());
+
         Vrfs vrfOriginal = bgpUtil.getVrfFromRd(rd);
         if (vrfOriginal == null) {
             LOG.error("delVrf: no vrf with existing rd {}. step aborted", rd);
@@ -2588,10 +2876,9 @@ public class BgpConfigurationManager {
 
         InstanceIdentifier<Vrfs> iid = iib.build();
 
-        InstanceIdentifier.builder(Bgp.class).build()
-                .child(Multipath.class, new MultipathKey(adfBuilder.getAfi(), adfBuilder.getSafi()));
         @SuppressWarnings("static-access")
-        InstanceIdentifier<Bgp> iid6 =  InstanceIdentifier.create(Bgp.class);
+        InstanceIdentifier<Bgp> iid6 =  iid.builder(Bgp.class).build()
+                .child(Multipath.class, new MultipathKey(adfBuilder.getAfi(), adfBuilder.getSafi())).create(Bgp.class);
         InstanceIdentifierBuilder<Vrfs> iib3 = iid6.child(Vrfs.class, new VrfsKey(rd)).builder();
         InstanceIdentifier<Vrfs> iidFinal = iib3.build();
 
@@ -2617,6 +2904,7 @@ public class BgpConfigurationManager {
             }
         }
         if (adfListOriginal.isEmpty()) {
+            LOG.debug("delVrf: delete iid: {}", iidFinal);
             delete(iidFinal);
             return true;
         }
@@ -2638,7 +2926,7 @@ public class BgpConfigurationManager {
         update(iib.build(), dto);
     }
 
-    public void multipaths(String rd, int maxpath) {
+    public void setMultipaths(String rd, int maxpath) {
         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
                 InstanceIdentifier
                         .builder(Bgp.class)
@@ -2648,6 +2936,14 @@ public class BgpConfigurationManager {
         update(iib.build(), dto);
     }
 
+    public void delMultipaths(String rd) {
+        InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
+                InstanceIdentifier.builder(Bgp.class)
+                        .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
+        InstanceIdentifier<VrfMaxpath> iid = iib.build();
+        delete(iid);
+    }
+
     /*
     * Remove Stale Marked Routes after timer expiry.
     */
@@ -2837,19 +3133,19 @@ public class BgpConfigurationManager {
     }
 
     public boolean isBgpConnected() {
-        return bgpRouter.isBgpConnected();
+        return (bgpRouter == null) ? false : bgpRouter.isBgpConnected();
     }
 
     public long getLastConnectedTS() {
-        return bgpRouter.getLastConnectedTS();
+        return (bgpRouter == null) ? 0 : bgpRouter.getLastConnectedTS();
     }
 
     public long getConnectTS() {
-        return bgpRouter.getConnectTS();
+        return (bgpRouter == null) ? 0 : bgpRouter.getConnectTS();
     }
 
     public long getStartTS() {
-        return bgpRouter.getStartTS();
+        return (bgpRouter == null) ? 0 : bgpRouter.getStartTS();
     }
 
     public TTransport getTransport() {
@@ -2892,6 +3188,8 @@ public class BgpConfigurationManager {
             bgpAlarmsReference.get().init();
             bgpAlarmsTask = executor.scheduleAtFixedRate(bgpAlarmsReference.get(), 0, 60 * 1000, TimeUnit.MILLISECONDS);
             LOG.info("Bgp Alarms task scheduled for every minute.");
+        } else {
+            LOG.trace("Bgp Alarms task already scheduled for every minute.");
         }
     }
 
@@ -2907,6 +3205,11 @@ public class BgpConfigurationManager {
         return bgpAlarmsReference.get();
     }
 
+    public void getPeerStatus(String nbrIp, long nbrAsNum) throws
+            BgpRouterException, TException {
+        bgpRouter.getPeerStatus(nbrIp, nbrAsNum);
+    }
+
     private static String appendNextHopToPrefix(String prefix, String nextHop) {
         return prefix + ":" + nextHop;
     }