bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import static org.opendaylight.netvirt.bgpmanager.oam.BgpConstants.HISTORY_LIMIT;
  11 import static org.opendaylight.netvirt.bgpmanager.oam.BgpConstants.HISTORY_THRESHOLD;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.base.Preconditions;
  15 import com.google.common.net.InetAddresses;
  16 import com.google.common.util.concurrent.Futures;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18 import com.google.common.util.concurrent.ThreadFactoryBuilder;
  19 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  20 import io.netty.util.concurrent.GlobalEventExecutor;
  21 import java.lang.reflect.Constructor;
  22 import java.lang.reflect.InvocationTargetException;
  23 import java.net.InetAddress;
  24 import java.net.InetSocketAddress;
  25 import java.net.NetworkInterface;
  26 import java.net.SocketException;
  27 import java.util.ArrayList;
  28 import java.util.Collection;
  29 import java.util.Collections;
  30 import java.util.Enumeration;
  31 import java.util.HashMap;
  32 import java.util.Iterator;
  33 import java.util.List;
  34 import java.util.Map;
  35 import java.util.Objects;
  36 import java.util.Timer;
  37 import java.util.concurrent.Callable;
  38 import java.util.concurrent.ConcurrentHashMap;
  39 import java.util.concurrent.CountDownLatch;
  40 import java.util.concurrent.ExecutionException;
  41 import java.util.concurrent.Executors;
  42 import java.util.concurrent.Future;
  43 import java.util.concurrent.ScheduledExecutorService;
  44 import java.util.concurrent.ScheduledFuture;
  45 import java.util.concurrent.TimeUnit;
  46 import java.util.concurrent.TimeoutException;
  47 import java.util.concurrent.atomic.AtomicBoolean;
  48 import java.util.concurrent.atomic.AtomicInteger;
  49 import java.util.concurrent.atomic.AtomicReference;
  50 import javax.annotation.PreDestroy;
  51 import javax.inject.Inject;
  52 import javax.inject.Singleton;
  53 import org.apache.thrift.TApplicationException;
  54 import org.apache.thrift.TException;
  55 import org.apache.thrift.transport.TTransport;
  56 import org.eclipse.jdt.annotation.Nullable;
  57 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
  58 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  59 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  60 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  61 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  62 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  63 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  64 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  65 import org.opendaylight.genius.mdsalutil.NwConstants;
  66 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  67 import org.opendaylight.infrautils.metrics.MetricProvider;
  68 import org.opendaylight.mdsal.eos.binding.api.Entity;
  69 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipCandidateRegistration;
  70 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipListenerRegistration;
  71 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  72 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  73 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  74 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  75 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  76 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  77 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  78 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  79 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  80 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  81 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  82 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  83 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  84 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  85 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  86 import org.opendaylight.netvirt.bgpmanager.thrift.gen.protocol_type;
  87 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  88 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  89 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  90 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  91 import org.opendaylight.ovsdb.utils.mdsal.utils.TransactionHistory;
  92 import org.opendaylight.ovsdb.utils.mdsal.utils.TransactionType;
  93 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebfd.rev190219.BfdConfig;
  94 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebfd.rev190219.BfdConfigBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.AddressFamily;
  96 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  97 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
  98 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EbgpService;
  99 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EncapType;
 100 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorInput;
 101 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorOutput;
 102 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorOutputBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.TcpMd5SignaturePasswordType;
 104 //import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.*;
 105 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
 106 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
 108 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.DcgwTepList;
 110 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
 111 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
 113 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathContainer;
 115 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsContainer;
 116 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksContainer;
 117 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathContainer;
 118 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsContainer;
 119 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTep;
 120 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTepBuilder;
 121 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTepKey;
 122 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.Multipath;
 123 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.MultipathBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.MultipathKey;
 125 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.Neighbors;
 126 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.NeighborsBuilder;
 127 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.NeighborsKey;
 128 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamilies;
 129 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamiliesBuilder;
 130 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamiliesKey;
 131 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.EbgpMultihop;
 132 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.EbgpMultihopBuilder;
 133 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.UpdateSource;
 134 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.UpdateSourceBuilder;
 135 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.Networks;
 136 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.NetworksBuilder;
 137 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.NetworksKey;
 138 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpath;
 139 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpathBuilder;
 140 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpathKey;
 141 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.Vrfs;
 142 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.VrfsBuilder;
 143 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.VrfsKey;
 144 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.vrfs.AddressFamiliesVrf;
 145 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.vrfs.AddressFamiliesVrfBuilder;
 146 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.TcpSecurityOption;
 147 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOption;
 148 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOptionBuilder;
 149 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
 150 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
 151 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
 152 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 153 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 154 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.macvrfentries.MacVrfEntry;
 155 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 156 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 157 import org.opendaylight.yangtools.yang.binding.DataObject;
 158 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 159 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 160 import org.opendaylight.yangtools.yang.common.RpcError;
 161 import org.opendaylight.yangtools.yang.common.RpcResult;
 162 import org.opendaylight.yangtools.yang.common.RpcResultBuilder;
 163 import org.opendaylight.yangtools.yang.common.Uint32;
 164 import org.osgi.framework.BundleContext;
 165 import org.osgi.util.tracker.ServiceTracker;
 166 import org.slf4j.Logger;
 167 import org.slf4j.LoggerFactory;
 168
 169 @Singleton
 170 public class BgpConfigurationManager implements EbgpService {
 171     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 172
 173     // to have stale FIB map (RD, Prefix)
 174     //  number of seconds wait for route sync-up between ODL and BGP
 175     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 176     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 177     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 178     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 179     private static final String DEF_UPORT = "6644";
 180     private static final String DEF_CHOST = "255.255.255.255"; // Invalid Host IP
 181     private static final String DEF_CPORT = "7644";
 182     private static final String DEF_BGP_SDNC_MIP = "127.0.0.1";
 183     //vpnservice.bgp.thrift.bgp.mip is the MIP present with ODL. Here we open 6644 port
 184     private static final String BGP_SDNC_MIP = "vpnservice.bgp.thrift.bgp.mip";
 185     private static final String BGP_GR_RESTART_TIMER_PROPERTY = "vpnservice.bgp.gr.timer";
 186     private static final String BGP_KA_TIMER_PROPERTY = "vpnservice.bgp.ka.timer";
 187     private static final String BGP_HOLD_TIMER_PROPERTY = "vpnservice.bgp.hold.timer";
 188     private static final String BGP_EOR_DELAY_PROPERTY = "vpnservice.bgp.eordelay";
 189     private static final int DEF_BGP_KA_TIME = 60;
 190     private static final int DEF_BGP_HOLD_TIME = 180;
 191     private static final int DEF_BGP_GR_TIME = 4000;
 192     private static final int RESTART_DEFAULT_GR = 90;
 193     private static final int DS_RETRY_COUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
 194     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
 195     private static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 196     private static final String BGP_EOR_DELAY = "vpnservice.bgp.eordelay";
 197     private static final String DEF_BGP_EOR_DELAY = "1800";
 198     private static final String BGP_ENTITY_NAME = "bgp";
 199     private static final String ADD_WARN = "Config store updated; undo with Delete if needed.";
 200     private static final String DEL_WARN = "Config store updated; undo with Add if needed.";
 201     private static final String UPD_WARN = "Update operation not supported; Config store updated;"
 202             + " restore with another Update if needed.";
 203     private static long bgp_as_num = 0;
 204     private static List<Neighbors> nbrList = new ArrayList<>();
 205     private int bgpKaTime = 0;
 206     private int bgpHoldTime = 0;
 207     private int bgpGrRestartTime = 0;
 208
 209     private static final Class<?>[] REACTORS = {
 210         ConfigServerReactor.class, AsIdReactor.class,
 211         GracefulRestartReactor.class, LoggingReactor.class,
 212         NeighborsReactor.class, UpdateSourceReactor.class,
 213         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 214         NetworksReactor.class, VrfsReactor.class, BgpReactor.class,
 215         MultipathReactor.class, VrfMaxpathReactor.class, BfdConfigReactor.class
 216     };
 217
 218     private IBgpManager bgpManager;
 219     private final DataBroker dataBroker;
 220     private final FibDSWriter fibDSWriter;
 221     private final IVpnLinkService vpnLinkService;
 222     private final BundleContext bundleContext;
 223     private final BgpUtil bgpUtil;
 224     private volatile Bgp config;
 225     private final BgpRouter bgpRouter;
 226     private final BgpSyncHandle bgpSyncHandle = new BgpSyncHandle();
 227     private volatile BgpThriftService bgpThriftService = null;
 228     private final int delayEorSeconds;
 229
 230     private final CountDownLatch initer = new CountDownLatch(1);
 231
 232     private final String hostStartup;
 233     private final String portStartup;
 234
 235     private final AtomicReference<BgpCounters> bgpCountersReference = new AtomicReference<>();
 236     private ScheduledFuture<?> bgpCountersTask;
 237
 238     private final AtomicReference<BgpAlarms> bgpAlarmsReference = new AtomicReference<>();
 239     private ScheduledFuture<?> bgpAlarmsTask;
 240
 241     private Future<?> lastReplayJobFt;
 242     private ScheduledFuture<?> routeCleanupFuture;
 243
 244     private long staleStartTime;
 245     private long staleEndTime;
 246     private long cfgReplayStartTime;
 247     private long cfgReplayEndTime;
 248     private long staleCleanupTime;
 249     private int totalStaledCount;
 250     private int totalCleared;
 251     private int totalExternalRoutes;
 252     private int totalExternalMacRoutes;
 253
 254     private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(
 255             new ThreadFactoryBuilder().setNameFormat("bgp-config-%d").setDaemon(true).build());
 256
 257     /**
 258      * this map store the new address families to send to quagga. When it is sended you must clear it.
 259      * The keys String are rd (route distinguisher).
 260      * */
 261     private final ConcurrentHashMap<String, List<AddressFamiliesVrf>> mapNewAdFamily = new ConcurrentHashMap<>();
 262
 263     // map<rd, map<prefix/len:nexthop, label>>
 264     private final Map<String, Map<String, Uint32>> staledFibEntriesMap = new ConcurrentHashMap<>();
 265
 266     // map<rd, map<tep-ip, map<mac, l2vni>>>
 267     private final Map<String, Map<String, Map<String, Uint32>>> rt2TepMap = new ConcurrentHashMap<>();
 268
 269     //map<rd+prefix/plen, list (nexthop)>
 270     private final Map<String,List> fibMap = new HashMap<>();
 271
 272     private final List<AutoCloseable> listeners = new ArrayList<>();
 273
 274     private final EntityOwnershipUtils entityOwnershipUtils;
 275     private final EntityOwnershipCandidateRegistration candidateRegistration;
 276     private final EntityOwnershipListenerRegistration entityListenerRegistration;
 277     private final MetricProvider metricProvider;
 278     private final TransactionHistory bgpUpdatesHistory;
 279
 280     private volatile AtomicBoolean eorSupressedDuetoUpgradeFlag = new AtomicBoolean(false);
 281
 282     @Inject
 283     public BgpConfigurationManager(final DataBroker dataBroker,
 284             final EntityOwnershipService entityOwnershipService,
 285             final FibDSWriter fibDSWriter,
 286             final IVpnLinkService vpnLinkSrvce,
 287             final BundleContext bundleContext,
 288             final BgpUtil bgpUtil,
 289             final MetricProvider metricProvider) {
 290         this.dataBroker = dataBroker;
 291         this.fibDSWriter = fibDSWriter;
 292         this.vpnLinkService = vpnLinkSrvce;
 293         this.bundleContext = bundleContext;
 294         this.bgpUtil = bgpUtil;
 295         this.metricProvider = metricProvider;
 296         hostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 297         portStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 298         bgpKaTime =
 299                 Integer.parseInt(getProperty(BGP_KA_TIMER_PROPERTY,
 300                         Integer.toString(DEF_BGP_KA_TIME)));
 301         bgpHoldTime =
 302                 Integer.parseInt(getProperty(BGP_HOLD_TIMER_PROPERTY,
 303                         Integer.toString(DEF_BGP_HOLD_TIME)));
 304         bgpGrRestartTime =
 305                 Integer.parseInt(getProperty(BGP_GR_RESTART_TIMER_PROPERTY,
 306                         Integer.toString(DEF_BGP_GR_TIME)));
 307         LOG.info("ConfigServer at {}:{}", hostStartup, portStartup);
 308         VtyshCli.setHostAddr(hostStartup);
 309         ClearBgpCli.setHostAddr(hostStartup);
 310         bgpUpdatesHistory = new TransactionHistory(HISTORY_LIMIT, HISTORY_THRESHOLD);
 311         bgpRouter = BgpRouter.newInstance(this::getConfig, this::isBGPEntityOwner, bgpUpdatesHistory);
 312         delayEorSeconds = Integer.parseInt(getProperty(BGP_EOR_DELAY, DEF_BGP_EOR_DELAY));
 313
 314         entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 315
 316         candidateRegistration = registerEntityCandidate(entityOwnershipService);
 317         entityListenerRegistration = registerEntityListener(entityOwnershipService);
 318
 319         LOG.info("BGP Configuration manager initialized");
 320         initer.countDown();
 321
 322         GlobalEventExecutor.INSTANCE.execute(() -> {
 323             ServiceTracker<IBgpManager, ?> tracker = null;
 324             try {
 325                 tracker = new ServiceTracker<>(bundleContext, IBgpManager.class, null);
 326                 tracker.open();
 327                 bgpManager = (IBgpManager) tracker.waitForService(TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES));
 328                 Preconditions.checkState(bgpManager != null, "IBgpManager service not found");
 329             } catch (InterruptedException e) {
 330                 throw new IllegalStateException("Error retrieving IBgpManager service", e);
 331             } finally {
 332                 if (tracker != null) {
 333                     tracker.close();
 334                 }
 335             }
 336
 337             String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
 338             if (InetAddresses.isInetAddress(getBgpSdncMipIp())) {
 339                 InetSocketAddress bgpThriftServerSocketAddr = new InetSocketAddress(getBgpSdncMipIp(),
 340                         Integer.parseInt(updatePort));
 341                 bgpThriftService = new BgpThriftService(bgpThriftServerSocketAddr, bgpManager, this);
 342                 if (isBGPEntityOwner()) {
 343                     //I am EoS owner of BGP, opening bgp thrift UPDATE-SERVER port.
 344                     LOG.info("BGP Configuration manager initialized: UPDATE-SERVER started");
 345                     bgpThriftService.start();
 346                 }
 347                 LOG.info("UPDATE server started :ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 348             } else {
 349                 LOG.error("Failed to init UPDATE server invalid ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 350             }
 351         });
 352         registerCallbacks();
 353         LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 354     }
 355
 356     public String getBgpSdncMipIp() {
 357         return getProperty(BGP_SDNC_MIP, DEF_BGP_SDNC_MIP);
 358     }
 359
 360     public long getStaleCleanupTime() {
 361         return staleCleanupTime;
 362     }
 363
 364     public void setStaleCleanupTime(long staleCleanupTime) {
 365         this.staleCleanupTime = staleCleanupTime;
 366     }
 367
 368     public long getCfgReplayEndTime() {
 369         return cfgReplayEndTime;
 370     }
 371
 372     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 373         this.cfgReplayEndTime = cfgReplayEndTime;
 374     }
 375
 376     public TransactionHistory getBgpUpdatesHistory() {
 377         return bgpUpdatesHistory;
 378     }
 379
 380     public long getCfgReplayStartTime() {
 381         return cfgReplayStartTime;
 382     }
 383
 384     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 385         this.cfgReplayStartTime = cfgReplayStartTime;
 386     }
 387
 388     public long getStaleEndTime() {
 389         return staleEndTime;
 390     }
 391
 392     public void setStaleEndTime(long staleEndTime) {
 393         this.staleEndTime = staleEndTime;
 394     }
 395
 396     public long getStaleStartTime() {
 397         return staleStartTime;
 398     }
 399
 400     public void setStaleStartTime(long staleStartTime) {
 401         this.staleStartTime = staleStartTime;
 402     }
 403
 404     private Object createListener(Class<?> cls) {
 405         try {
 406             Constructor<?> ctor = cls.getConstructor(BgpConfigurationManager.class);
 407             return ctor.newInstance(this);
 408         } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException
 409                 e) {
 410             LOG.error("Failed to create listener object", e);
 411             return null;
 412         }
 413     }
 414
 415     private void registerCallbacks() {
 416         for (Class<?> reactor : REACTORS) {
 417             Object obj = createListener(reactor);
 418             if (obj != null) {
 419                 AsyncDataTreeChangeListenerBase dcl = (AsyncDataTreeChangeListenerBase) obj;
 420                 dcl.registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 421                 listeners.add(dcl);
 422             }
 423         }
 424     }
 425
 426     @SuppressWarnings("checkstyle:IllegalCatch")
 427     @PreDestroy
 428     public void close() {
 429         executor.shutdown();
 430
 431         if (bgpThriftService != null) {
 432             bgpThriftService.stop();
 433             bgpThriftService = null;
 434         }
 435
 436         if (isBgpConnected()) {
 437             //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 438             bgpRouter.disconnect();
 439         }
 440
 441         if (candidateRegistration != null) {
 442             candidateRegistration.close();
 443         }
 444
 445         entityListenerRegistration.close();
 446
 447         listeners.forEach(l -> {
 448             try {
 449                 l.close();
 450             } catch (Exception e) {
 451                 LOG.warn("Error closing {}", l ,e);
 452             }
 453         });
 454
 455         LOG.info("{} close", getClass().getSimpleName());
 456     }
 457
 458     private String getProperty(String var, String def) {
 459         String property = bundleContext.getProperty(var);
 460         return property == null ? def : property;
 461     }
 462
 463     private EntityOwnershipCandidateRegistration registerEntityCandidate(
 464             final EntityOwnershipService entityOwnershipService) {
 465         try {
 466             return entityOwnershipService.registerCandidate(
 467                     new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME));
 468         } catch (CandidateAlreadyRegisteredException e) {
 469             LOG.error("failed to register bgp entity", e);
 470             return null;
 471         }
 472     }
 473
 474     private EntityOwnershipListenerRegistration registerEntityListener(
 475             final EntityOwnershipService entityOwnershipService) {
 476         return entityOwnershipService.registerListener(BGP_ENTITY_TYPE_FOR_OWNERSHIP, ownershipChange -> {
 477             LOG.info("entity owner change event fired: {}", ownershipChange);
 478
 479             if (ownershipChange.getState() == EntityOwnershipChangeState.LOCAL_OWNERSHIP_GRANTED) {
 480                 LOG.info("This PL is the Owner");
 481                 if (bgpThriftService != null) {
 482                     //opening UPDATE-SERVER port.
 483                     bgpThriftService.start();
 484                 } else {
 485                     LOG.error("I am the owner of BGP entity, but bgpThriftService is not initialized yet");
 486                 }
 487                 bgpRestarted();
 488             } else {
 489                 LOG.info("Not owner: hasOwner: {}, isOwner: {}", ownershipChange.getState().hasOwner(),
 490                         ownershipChange.getState().isOwner());
 491                 if (bgpThriftService != null && bgpThriftService.isBgpThriftServiceStarted()) {
 492                     //close the bgp Thrift Update-SERVER port opened on non-Entity Owner
 493                     bgpThriftService.stop();
 494                 }
 495                 if (isBgpConnected()) {
 496                     //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 497                     bgpRouter.disconnect();
 498                 }
 499             }
 500         });
 501     }
 502
 503     public boolean isBGPEntityOwner() {
 504         if (entityOwnershipUtils == null) {
 505             LOG.error("entityOwnershipUtils is NULL when listener callbacks fired");
 506             return false;
 507         }
 508         return entityOwnershipUtils.isEntityOwner(new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME), 0, 1);
 509     }
 510
 511     public Bgp get() {
 512         config = getConfig();
 513         return config;
 514     }
 515
 516     public class ConfigServerReactor
 517             extends AsyncDataTreeChangeListenerBase<ConfigServer, ConfigServerReactor>
 518             implements ClusteredDataTreeChangeListener<ConfigServer> {
 519         private static final String YANG_OBJ = "config-server ";
 520
 521         public ConfigServerReactor() {
 522             super(ConfigServer.class, ConfigServerReactor.class);
 523         }
 524
 525         @Override
 526         protected void add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 527             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 528             if (!isBGPEntityOwner()) {
 529                 return;
 530             }
 531
 532             try {
 533                 initer.await();
 534             } catch (InterruptedException e) {
 535                 // Ignored
 536             }
 537             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 538             bgpRouter.configServerUpdated();
 539             synchronized (BgpConfigurationManager.this) {
 540                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 541                         val.getPort().intValue());
 542                 if (!res) {
 543                     LOG.error(YANG_OBJ + "Add failed; " + ADD_WARN);
 544                 }
 545             }
 546             VtyshCli.setHostAddr(val.getHost().getValue());
 547             ClearBgpCli.setHostAddr(val.getHost().getValue());
 548         }
 549
 550         @Override
 551         protected ConfigServerReactor getDataTreeChangeListener() {
 552             return ConfigServerReactor.this;
 553         }
 554
 555         @Override
 556         protected InstanceIdentifier<ConfigServer> getWildCardPath() {
 557             return InstanceIdentifier.create(Bgp.class).child(ConfigServer.class);
 558         }
 559
 560         @Override
 561         protected void remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 562             LOG.trace("received bgp disconnect");
 563             if (!isBGPEntityOwner()) {
 564                 return;
 565             }
 566
 567             bgpRouter.configServerUpdated();
 568
 569             synchronized (BgpConfigurationManager.this) {
 570                 if (bgp_as_num != 0) {
 571                     try {
 572                         bgpRouter.stopBgp(bgp_as_num);
 573                         stopBgpCountersTask();
 574                         stopBgpAlarmsTask();
 575                     } catch (TException | BgpRouterException e) {
 576                         LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 577                     }
 578                 } else {
 579                     LOG.debug("bgp as-id is null while removing config-server");
 580                 }
 581                 bgpRouter.disconnect();
 582             }
 583         }
 584
 585         @Override
 586         protected void update(InstanceIdentifier<ConfigServer> iid,
 587                 ConfigServer oldval, ConfigServer newval) {
 588             LOG.trace("received bgp Connection update");
 589             if (!isBGPEntityOwner()) {
 590                 return;
 591             }
 592             LOG.error(YANG_OBJ + UPD_WARN);
 593         }
 594     }
 595
 596     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
 597             justification = "https://github.com/spotbugs/spotbugs/issues/811")
 598     private BgpRouter getClient(String yangObj) {
 599         if (bgpRouter == null || !bgpRouter.isBgpConnected()) {
 600             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 601             return null;
 602         }
 603         return bgpRouter;
 604     }
 605
 606     public class AsIdReactor
 607             extends AsyncDataTreeChangeListenerBase<AsId, AsIdReactor>
 608             implements ClusteredDataTreeChangeListener<AsId> {
 609
 610         private static final String YANG_OBJ = "as-id ";
 611
 612         public AsIdReactor() {
 613             super(AsId.class, AsIdReactor.class);
 614         }
 615
 616         @Override
 617         protected void add(InstanceIdentifier<AsId> iid, AsId val) {
 618             LOG.error("received bgp add asid {}", val);
 619             if (!isBGPEntityOwner()) {
 620                 return;
 621             }
 622             LOG.debug("received add router config asNum {}", val.getLocalAs());
 623             bgp_as_num = val.getLocalAs().longValue();
 624             synchronized (BgpConfigurationManager.this) {
 625                 BgpRouter br = getClient(YANG_OBJ);
 626                 if (br == null) {
 627                     LOG.debug("{} Unable to process add for asNum {}; {} {}", YANG_OBJ, val.getLocalAs(),
 628                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 629                     return;
 630                 }
 631                 bgpRestarted();
 632
 633                 startBgpCountersTask();
 634                 startBgpAlarmsTask();
 635             }
 636         }
 637
 638         @Override
 639         protected AsIdReactor getDataTreeChangeListener() {
 640             return AsIdReactor.this;
 641         }
 642
 643         @Override
 644         protected InstanceIdentifier<AsId> getWildCardPath() {
 645             return InstanceIdentifier.create(Bgp.class).child(AsId.class);
 646         }
 647
 648         @Override
 649         protected void remove(InstanceIdentifier<AsId> iid, AsId val) {
 650             LOG.error("received delete router config asNum {}", val.getLocalAs());
 651             if (!isBGPEntityOwner()) {
 652                 return;
 653             }
 654             synchronized (BgpConfigurationManager.this) {
 655                 long asNum = val.getLocalAs().toJava();
 656                 BgpRouter br = getClient(YANG_OBJ);
 657                 bgp_as_num = 0;
 658                 if (br == null) {
 659                     LOG.debug("{} Unable to process remove for asNum {}; {} {}", YANG_OBJ, asNum,
 660                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 661                     return;
 662                 }
 663                 try {
 664                     br.stopBgp(asNum);
 665                 } catch (TException | BgpRouterException e) {
 666                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 667                 }
 668
 669                 stopBgpCountersTask();
 670                 stopBgpAlarmsTask();
 671
 672                 Bgp conf = getConfig();
 673                 if (conf == null) {
 674                     LOG.error("Config Null while removing the as-id");
 675                     return;
 676                 }
 677                 LOG.debug("Removing external routes from FIB");
 678                 deleteExternalFibRoutes();
 679                 List<Neighbors> nbrs = conf.getNeighborsContainer() == null ? null
 680                         : conf.getNeighborsContainer().getNeighbors();
 681                 if (nbrs != null && nbrs.size() > 0) {
 682                     LOG.error("Tring to remove the as-id when neighbor config is already present");
 683                     for (Neighbors nbr : nbrs) {
 684                         LOG.debug("Removing Neighbor {} from Data store", nbr.getAddress().getValue());
 685                         delNeighbor(nbr.getAddress().getValue());
 686                     }
 687                 }
 688             }
 689         }
 690
 691         @Override
 692         protected void update(InstanceIdentifier<AsId> iid,
 693                 AsId oldval, AsId newval) {
 694             if (!isBGPEntityOwner()) {
 695                 return;
 696             }
 697             LOG.error(YANG_OBJ + UPD_WARN);
 698         }
 699     }
 700
 701     public class GracefulRestartReactor
 702             extends AsyncDataTreeChangeListenerBase<GracefulRestart, GracefulRestartReactor>
 703             implements ClusteredDataTreeChangeListener<GracefulRestart> {
 704
 705         private static final String YANG_OBJ = "graceful-restart ";
 706
 707         public GracefulRestartReactor() {
 708             super(GracefulRestart.class, GracefulRestartReactor.class);
 709         }
 710
 711         @Override
 712         protected void add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 713             if (!isBGPEntityOwner()) {
 714                 return;
 715             }
 716             synchronized (BgpConfigurationManager.this) {
 717                 int stalePathTime = val.getStalepathTime().intValue();
 718                 BgpRouter br = getClient(YANG_OBJ);
 719                 if (br == null) {
 720                     LOG.error("{} Unable to add stale-path time {}; {} {}", YANG_OBJ, stalePathTime,
 721                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 722                     return;
 723                 }
 724                 try {
 725                     br.addGracefulRestart(stalePathTime);
 726                 } catch (TException | BgpRouterException e) {
 727                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 728                 }
 729             }
 730         }
 731
 732         @Override
 733         protected GracefulRestartReactor getDataTreeChangeListener() {
 734             return GracefulRestartReactor.this;
 735         }
 736
 737         @Override
 738         protected InstanceIdentifier<GracefulRestart> getWildCardPath() {
 739             return InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
 740         }
 741
 742         @Override
 743         protected void remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 744             if (!isBGPEntityOwner()) {
 745                 return;
 746             }
 747             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 748             synchronized (BgpConfigurationManager.this) {
 749                 BgpRouter br = getClient(YANG_OBJ);
 750                 if (br == null) {
 751                     LOG.error("{} Unable to delete stale-path time; {} {}", YANG_OBJ,
 752                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 753                     return;
 754                 }
 755                 try {
 756                     br.delGracefulRestart();
 757                 } catch (TException | BgpRouterException e) {
 758                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 759                 }
 760             }
 761         }
 762
 763         @Override
 764         protected void update(InstanceIdentifier<GracefulRestart> iid,
 765                 GracefulRestart oldval, GracefulRestart newval) {
 766             if (!isBGPEntityOwner()) {
 767                 return;
 768             }
 769             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 770             synchronized (BgpConfigurationManager.this) {
 771                 int stalePathTime = newval.getStalepathTime().intValue();
 772                 BgpRouter br = getClient(YANG_OBJ);
 773                 if (br == null) {
 774                     LOG.error("{} Unable to update stale-path time to {}; {} {}", YANG_OBJ, stalePathTime,
 775                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 776                     return;
 777                 }
 778                 try {
 779                     br.addGracefulRestart(stalePathTime);
 780                 } catch (TException | BgpRouterException e) {
 781                     LOG.error("{} update received exception; {}", YANG_OBJ, ADD_WARN, e);
 782                 }
 783             }
 784         }
 785     }
 786
 787     public class LoggingReactor
 788             extends AsyncDataTreeChangeListenerBase<Logging, LoggingReactor>
 789             implements ClusteredDataTreeChangeListener<Logging> {
 790
 791         private static final String YANG_OBJ = "logging ";
 792
 793         public LoggingReactor() {
 794             super(Logging.class, LoggingReactor.class);
 795         }
 796
 797         @Override
 798         protected void add(InstanceIdentifier<Logging> iid, Logging val) {
 799             if (!isBGPEntityOwner()) {
 800                 return;
 801             }
 802             synchronized (BgpConfigurationManager.this) {
 803                 BgpRouter br = getClient(YANG_OBJ);
 804                 if (br == null) {
 805                     LOG.error("{} Unable to add logging for qbgp; {} {}", YANG_OBJ,
 806                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 807                     return;
 808                 }
 809                 try {
 810                     br.setLogging(val.getFile(), val.getLevel());
 811                 } catch (TException | BgpRouterException e) {
 812                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 813                 }
 814             }
 815         }
 816
 817         @Override
 818         protected LoggingReactor getDataTreeChangeListener() {
 819             return LoggingReactor.this;
 820         }
 821
 822         @Override
 823         protected InstanceIdentifier<Logging> getWildCardPath() {
 824             return InstanceIdentifier.create(Bgp.class).child(Logging.class);
 825         }
 826
 827         @Override
 828         protected void remove(InstanceIdentifier<Logging> iid, Logging val) {
 829             if (!isBGPEntityOwner()) {
 830                 return;
 831             }
 832             LOG.debug("received remove Logging config val {}", val.getLevel());
 833             synchronized (BgpConfigurationManager.this) {
 834                 BgpRouter br = getClient(YANG_OBJ);
 835                 if (br == null) {
 836                     LOG.error("{} Unable to remove logging for qbgp; {} {}", YANG_OBJ,
 837                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 838                     return;
 839                 }
 840                 try {
 841                     br.setLogging(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
 842                 } catch (TException | BgpRouterException e) {
 843                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 844                 }
 845             }
 846         }
 847
 848         @Override
 849         protected void update(InstanceIdentifier<Logging> iid,
 850                 Logging oldval, Logging newval) {
 851             if (!isBGPEntityOwner()) {
 852                 return;
 853             }
 854             synchronized (BgpConfigurationManager.this) {
 855                 BgpRouter br = getClient(YANG_OBJ);
 856                 if (br == null) {
 857                     LOG.error("{} Unable to update logging for qbgp; {} {}", YANG_OBJ,
 858                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 859                     return;
 860                 }
 861                 try {
 862                     br.setLogging(newval.getFile(), newval.getLevel());
 863                 } catch (TException | BgpRouterException e) {
 864                     LOG.error("{} newval received exception; {}", YANG_OBJ, ADD_WARN, e);
 865                 }
 866             }
 867         }
 868     }
 869
 870     public class NeighborsReactor
 871             extends AsyncDataTreeChangeListenerBase<Neighbors, NeighborsReactor>
 872             implements ClusteredDataTreeChangeListener<Neighbors> {
 873
 874         private static final String YANG_OBJ = "neighbors ";
 875
 876         public NeighborsReactor() {
 877             super(Neighbors.class, NeighborsReactor.class);
 878         }
 879
 880         @Override
 881         protected void add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 882             if (nbrList != null && !nbrList.contains(val)) {
 883                 LOG.trace("Adding nbr {} to nbrlist", val.getAddress().getValue());
 884                 nbrList.add(val);
 885             }
 886             if (!isBGPEntityOwner()) {
 887                 return;
 888             }
 889             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 890             synchronized (BgpConfigurationManager.this) {
 891                 String peerIp = val.getAddress().getValue();
 892                 String sourceIp = (val.getUpdateSource() == null) ? null :
 893                                     val.getUpdateSource().getSourceIp().getValue();
 894                 int nhops = (val.getEbgpMultihop() == null) ? 0 :
 895                                     val.getEbgpMultihop().getNhops().intValue();
 896                 List<AddressFamilies> afs = val.getAddressFamilies();
 897                 long as = val.getRemoteAs().toJava();
 898                 final String md5Secret = extractMd5Secret(val);
 899                 BgpRouter br = getClient(YANG_OBJ);
 900                 if (br == null) {
 901                     LOG.debug("{} Unable to process add for peer {} as {}; {} {}", YANG_OBJ, peerIp, as,
 902                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 903                     return;
 904                 }
 905                 try {
 906                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 907                     br.addNeighbor(peerIp, as, md5Secret);
 908                     if (nhops != 0) {
 909                         br.addEbgpMultihop(peerIp, nhops);
 910                     }
 911                     if (sourceIp != null) {
 912                         br.addUpdateSource(peerIp, sourceIp);
 913                     }
 914                     if (afs != null) {
 915                         for (AddressFamilies af : afs) {
 916                             af_afi afi = af_afi.findByValue(af.getAfi().intValue());
 917                             af_safi safi = af_safi.findByValue(af.getSafi().intValue());
 918                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
 919                         }
 920                     }
 921
 922                 } catch (TException | BgpRouterException e) {
 923                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 924                 }
 925             }
 926         }
 927
 928         @Override
 929         protected NeighborsReactor getDataTreeChangeListener() {
 930             return NeighborsReactor.this;
 931         }
 932
 933         @Override
 934         protected InstanceIdentifier<Neighbors> getWildCardPath() {
 935             return InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class);
 936         }
 937
 938         @Override
 939         protected void remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 940             if (nbrList != null && nbrList.contains(val)) {
 941                 LOG.trace("Removing nbr {} from nbr list", val.getAddress().getValue());
 942                 nbrList.remove(val);
 943             }
 944             if (!isBGPEntityOwner()) {
 945                 return;
 946             }
 947             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 948             synchronized (BgpConfigurationManager.this) {
 949                 String peerIp = val.getAddress().getValue();
 950                 BgpRouter br = getClient(YANG_OBJ);
 951                 if (br == null) {
 952                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 953                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 954                     return;
 955                 }
 956                 try {
 957                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 958                     br.delNeighbor(peerIp);
 959                 } catch (TException | BgpRouterException e) {
 960                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 961                 }
 962
 963                 final BgpAlarms bgpAlarms = getBgpAlarms();
 964                 if (bgpAlarms != null) {
 965                     bgpAlarms.clearBgpNbrDownAlarm(peerIp);
 966                 }
 967
 968                 if (bgpUtil.isBfdEnabled()) {
 969                     final BgpCounters bgpCounters = getBgpCounters();
 970                     if (bgpCounters != null) {
 971                         bgpCounters.clearBfdNbrCounters(peerIp);
 972                     }
 973                 }
 974             }
 975         }
 976
 977         @Override
 978         protected void update(InstanceIdentifier<Neighbors> iid,
 979                 Neighbors oldval, Neighbors newval) {
 980             if (!isBGPEntityOwner()) {
 981                 return;
 982             }
 983             //purposefully nothing to do.
 984         }
 985     }
 986
 987     public class EbgpMultihopReactor
 988             extends AsyncDataTreeChangeListenerBase<EbgpMultihop, EbgpMultihopReactor>
 989             implements ClusteredDataTreeChangeListener<EbgpMultihop> {
 990
 991         private static final String YANG_OBJ = "ebgp-multihop ";
 992
 993         public EbgpMultihopReactor() {
 994             super(EbgpMultihop.class, EbgpMultihopReactor.class);
 995         }
 996
 997         @Override
 998         protected void add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 999             if (!isBGPEntityOwner()) {
1000                 return;
1001             }
1002             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
1003             synchronized (BgpConfigurationManager.this) {
1004                 String peerIp = val.getPeerIp().getValue();
1005                 BgpRouter br = getClient(YANG_OBJ);
1006                 if (br == null) {
1007                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1008                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1009                     return;
1010                 }
1011                 try {
1012                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
1013                 } catch (TException | BgpRouterException e) {
1014                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1015                 }
1016             }
1017         }
1018
1019         @Override
1020         protected EbgpMultihopReactor getDataTreeChangeListener() {
1021             return EbgpMultihopReactor.this;
1022         }
1023
1024         @Override
1025         protected InstanceIdentifier<EbgpMultihop> getWildCardPath() {
1026             return InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
1027                     .child(EbgpMultihop.class);
1028         }
1029
1030         @Override
1031         protected void remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
1032             if (!isBGPEntityOwner()) {
1033                 return;
1034             }
1035             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
1036             synchronized (BgpConfigurationManager.this) {
1037                 String peerIp = val.getPeerIp().getValue();
1038                 BgpRouter br = getClient(YANG_OBJ);
1039                 if (br == null) {
1040                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1041                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1042                     return;
1043                 }
1044                 try {
1045                     br.delEbgpMultihop(peerIp);
1046                 } catch (TException | BgpRouterException e) {
1047                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1048                 }
1049             }
1050         }
1051
1052         @Override
1053         protected void update(InstanceIdentifier<EbgpMultihop> iid,
1054                 EbgpMultihop oldval, EbgpMultihop newval) {
1055             if (!isBGPEntityOwner()) {
1056                 return;
1057             }
1058             LOG.error(YANG_OBJ + UPD_WARN);
1059         }
1060     }
1061
1062     public class UpdateSourceReactor
1063             extends AsyncDataTreeChangeListenerBase<UpdateSource, UpdateSourceReactor>
1064             implements ClusteredDataTreeChangeListener<UpdateSource> {
1065
1066         private static final String YANG_OBJ = "update-source ";
1067
1068         public UpdateSourceReactor() {
1069             super(UpdateSource.class, UpdateSourceReactor.class);
1070         }
1071
1072         @Override
1073         protected void add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
1074             if (!isBGPEntityOwner()) {
1075                 return;
1076             }
1077             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
1078             synchronized (BgpConfigurationManager.this) {
1079                 String peerIp = val.getPeerIp().getValue();
1080                 BgpRouter br = getClient(YANG_OBJ);
1081                 if (br == null) {
1082                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1083                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1084                     return;
1085                 }
1086                 try {
1087                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
1088                 } catch (TException | BgpRouterException e) {
1089                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1090                 }
1091             }
1092         }
1093
1094         @Override
1095         protected UpdateSourceReactor getDataTreeChangeListener() {
1096             return UpdateSourceReactor.this;
1097         }
1098
1099         @Override
1100         protected InstanceIdentifier<UpdateSource> getWildCardPath() {
1101             return InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
1102                     .child(UpdateSource.class);
1103         }
1104
1105         @Override
1106         protected void remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
1107             if (!isBGPEntityOwner()) {
1108                 return;
1109             }
1110             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
1111             synchronized (BgpConfigurationManager.this) {
1112                 String peerIp = val.getPeerIp().getValue();
1113                 BgpRouter br = getClient(YANG_OBJ);
1114                 if (br == null) {
1115                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1116                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1117                     return;
1118                 }
1119                 try {
1120                     br.delUpdateSource(peerIp);
1121                 } catch (TException | BgpRouterException e) {
1122                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1123                 }
1124             }
1125         }
1126
1127         @Override
1128         protected void update(InstanceIdentifier<UpdateSource> iid,
1129                 UpdateSource oldval, UpdateSource newval) {
1130             if (!isBGPEntityOwner()) {
1131                 return;
1132             }
1133             LOG.error(YANG_OBJ + UPD_WARN);
1134         }
1135     }
1136
1137     public class AddressFamiliesReactor
1138             extends AsyncDataTreeChangeListenerBase<AddressFamilies, AddressFamiliesReactor>
1139             implements ClusteredDataTreeChangeListener<AddressFamilies> {
1140
1141         private static final String YANG_OBJ = "address-families ";
1142
1143         public AddressFamiliesReactor() {
1144             super(AddressFamilies.class, AddressFamiliesReactor.class);
1145         }
1146
1147         @Override
1148         protected void add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1149             if (!isBGPEntityOwner()) {
1150                 return;
1151             }
1152             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
1153             synchronized (BgpConfigurationManager.this) {
1154                 String peerIp = val.getPeerIp().getValue();
1155                 BgpRouter br = getClient(YANG_OBJ);
1156                 if (br == null) {
1157                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1158                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1159                     return;
1160                 }
1161                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1162                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1163                 try {
1164                     br.addAddressFamily(peerIp, afi, safi);
1165                 } catch (TException | BgpRouterException e) {
1166                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1167                 }
1168             }
1169         }
1170
1171         @Override
1172         protected AddressFamiliesReactor getDataTreeChangeListener() {
1173             return AddressFamiliesReactor.this;
1174         }
1175
1176         @Override
1177         protected InstanceIdentifier<AddressFamilies> getWildCardPath() {
1178             return InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
1179                     .child(AddressFamilies.class);
1180         }
1181
1182         @Override
1183         protected void remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1184             if (!isBGPEntityOwner()) {
1185                 return;
1186             }
1187             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
1188             synchronized (BgpConfigurationManager.this) {
1189                 String peerIp = val.getPeerIp().getValue();
1190                 BgpRouter br = getClient(YANG_OBJ);
1191                 if (br == null) {
1192                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1193                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1194                     return;
1195                 }
1196                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1197                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1198                 try {
1199                     br.delAddressFamily(peerIp, afi, safi);
1200                 } catch (TException | BgpRouterException e) {
1201                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1202                 }
1203             }
1204         }
1205
1206         @Override
1207         protected void update(InstanceIdentifier<AddressFamilies> iid,
1208                 AddressFamilies oldval, AddressFamilies newval) {
1209             if (!isBGPEntityOwner()) {
1210                 return;
1211             }
1212             LOG.error(YANG_OBJ + UPD_WARN);
1213         }
1214     }
1215
1216     public class NetworksReactor
1217             extends AsyncDataTreeChangeListenerBase<Networks, NetworksReactor>
1218             implements ClusteredDataTreeChangeListener<Networks> {
1219
1220         private static final String YANG_OBJ = "networks ";
1221
1222         public NetworksReactor() {
1223             super(Networks.class, NetworksReactor.class);
1224         }
1225
1226         @Override
1227         public NetworksReactor getDataTreeChangeListener() {
1228             return NetworksReactor.this;
1229         }
1230
1231         @Override
1232         protected void add(InstanceIdentifier<Networks> iid, Networks val) {
1233             if (!isBGPEntityOwner()) {
1234                 return;
1235             }
1236             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1237             synchronized (BgpConfigurationManager.this) {
1238                 String rd = val.getRd();
1239                 String pfxlen = val.getPrefixLen();
1240                 String nh = val.getNexthop().getValue();
1241                 BgpRouter br = getClient(YANG_OBJ);
1242                 if (br == null) {
1243                     LOG.debug("{} Unable to process add for rd {} prefix {} nexthop {}; {} {}", YANG_OBJ, rd, pfxlen,
1244                             nh, BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1245                     return;
1246                 }
1247                 Long label = val.getLabel().toJava();
1248                 int lbl = label == null ? qbgpConstants.LBL_NO_LABEL
1249                         : label.intValue();
1250                 int l3vni = val.getL3vni() == null ? qbgpConstants.LBL_NO_LABEL
1251                         : val.getL3vni().intValue();
1252                 int l2vni = val.getL2vni() == null ? qbgpConstants.LBL_NO_LABEL
1253                         : val.getL2vni().intValue();
1254
1255                 BgpControlPlaneType protocolType = val.getBgpControlPlaneType();
1256                 int ethernetTag = val.getEthtag().intValue();
1257                 String esi = val.getEsi();
1258                 String macaddress = val.getMacaddress();
1259                 EncapType encapType = val.getEncapType();
1260                 String routerMac = val.getRoutermac();
1261
1262                 try {
1263                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni, BgpUtil.convertToThriftProtocolType(protocolType),
1264                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1265                 } catch (TException | BgpRouterException e) {
1266                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1267                 }
1268             }
1269         }
1270
1271         @Override
1272         protected InstanceIdentifier<Networks> getWildCardPath() {
1273             return InstanceIdentifier.create(Bgp.class).child(NetworksContainer.class).child(Networks.class);
1274         }
1275
1276         @Override
1277         protected void remove(InstanceIdentifier<Networks> iid, Networks val) {
1278             if (!isBGPEntityOwner()) {
1279                 return;
1280             }
1281             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1282             synchronized (BgpConfigurationManager.this) {
1283                 String rd = val.getRd();
1284                 String pfxlen = val.getPrefixLen();
1285                 BgpRouter br = getClient(YANG_OBJ);
1286                 if (br == null) {
1287                     LOG.debug("{} Unable to process remove for rd {} prefix {}; {} {}", YANG_OBJ, rd, pfxlen,
1288                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1289                     return;
1290                 }
1291                 Long label = val.getLabel().toJava();
1292                 int lbl = label == null ? 0 : label.intValue();
1293                 if (rd == null && lbl > 0) {
1294                     //LU prefix is being deleted.
1295                     rd = Integer.toString(lbl);
1296                 }
1297                 try {
1298                     br.delPrefix(rd, pfxlen);
1299                 } catch (TException | BgpRouterException e) {
1300                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1301                 }
1302             }
1303         }
1304
1305         /**get the value AFI from a prefix as "x.x.x.x/x".
1306          *
1307          * @param pfxlen the prefix to get an afi
1308          * @return the afi value as you are need
1309          */
1310         public  int  testValueAFI(String pfxlen) {
1311             int afiNew = af_afi.AFI_IP.getValue();
1312             try {
1313                 String ipOnly = pfxlen.substring(0, pfxlen.lastIndexOf("/"));
1314                 java.net.Inet6Address.getByName(ipOnly);
1315                 afiNew = af_afi.AFI_IPV6.getValue();
1316             } catch (java.net.UnknownHostException e) {
1317                 //ce n'est pas de l'ipv6
1318             }
1319             return afiNew;
1320         }
1321
1322
1323         @Override
1324         protected void update(final InstanceIdentifier<Networks> iid,
1325                 final Networks oldval, final Networks newval) {
1326             if (!isBGPEntityOwner()) {
1327                 return;
1328             }
1329             if (oldval.equals(newval)) {
1330                 //Update: OLD and New values are same, no need to trigger remove/add.
1331                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1332                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1333                 return;
1334             }
1335             LOG.debug("update networks old val RD: {}, Prefix: {}, Label: {}, NH: {} "
1336                             + "new val RD: {}, Prefix: {}, Label: {}, NH: {}",
1337                     oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop(),
1338                     newval.getRd(), newval.getPrefixLen(), newval.getLabel(), newval.getNexthop());
1339             remove(iid, oldval);
1340             add(iid, newval);
1341         }
1342     }
1343
1344     static Timer timer = new Timer();
1345
1346     public class VrfsReactor
1347             extends AsyncDataTreeChangeListenerBase<Vrfs, VrfsReactor>
1348             implements ClusteredDataTreeChangeListener<Vrfs> {
1349
1350         private static final String YANG_OBJ = "vrfs ";
1351
1352         public VrfsReactor() {
1353             super(Vrfs.class, VrfsReactor.class);
1354         }
1355
1356         @Override
1357         protected void add(InstanceIdentifier<Vrfs> iid, Vrfs vrfs) {
1358             if (!isBGPEntityOwner()) {
1359                 return;
1360             }
1361             LOG.debug("received add Vrfs config value {}", vrfs.getRd());
1362             synchronized (BgpConfigurationManager.this) {
1363                 String rd = vrfs.getRd();
1364                 BgpRouter br = getClient(YANG_OBJ);
1365                 if (br == null) {
1366                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1367                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1368                     return;
1369                 }
1370                 try {
1371                     List<AddressFamiliesVrf> vrfAddrFamilyList = vrfs.getAddressFamiliesVrf();
1372                     for (AddressFamiliesVrf vrfAddrFamily : vrfAddrFamilyList) {
1373                         /*add to br the new vrfs arguments*/
1374                         br.addVrf(BgpUtil.getLayerType(vrfAddrFamily), rd, vrfs.getImportRts(),
1375                                 vrfs.getExportRts(), vrfAddrFamily.getAfi().toJava(), vrfAddrFamily.getSafi().toJava());
1376                     }
1377                     /*add to br the vrfs contained in mapNewAdFamily*/
1378                     List<AddressFamiliesVrf> vrfAddrFamilyListFromMap = mapNewAdFamily.get(rd);
1379                     if (vrfAddrFamilyListFromMap == null) {
1380                         return;
1381                     }
1382
1383                     for (AddressFamiliesVrf adf : vrfAddrFamilyListFromMap) {
1384                         if (vrfAddrFamilyList.contains(adf)) {
1385                             mapNewAdFamily.remove(rd);
1386                         } else  if (adf != null) {
1387
1388                             br.addVrf(BgpUtil.getLayerType(adf), rd, vrfs.getImportRts(),
1389                                     vrfs.getExportRts(), adf.getAfi().toJava(), adf.getSafi().toJava());
1390                             // remove AddressFamiliesVrf which was already added to BGP
1391                             vrfAddrFamilyListFromMap.remove(adf);
1392                             if (vrfAddrFamilyListFromMap.isEmpty()) {
1393                                 // remove Vrf entry from temp mapNewAdFamily if all its AddressFamiliesVrf was
1394                                 // added to BGP
1395                                 mapNewAdFamily.remove(rd);
1396                             }
1397                         }
1398                     }
1399                 } catch (TException | BgpRouterException e) {
1400                     LOG.error("{} get {}, Add received exception", YANG_OBJ, ADD_WARN, e);
1401                 }
1402             }
1403         }
1404
1405         @Override
1406         protected VrfsReactor getDataTreeChangeListener() {
1407             return VrfsReactor.this;
1408         }
1409
1410         @Override
1411         protected InstanceIdentifier<Vrfs> getWildCardPath() {
1412             return InstanceIdentifier.create(Bgp.class).child(VrfsContainer.class).child(Vrfs.class);
1413         }
1414
1415         @Override
1416         protected void remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1417             if (!isBGPEntityOwner()) {
1418                 return;
1419             }
1420             LOG.debug("received remove Vrfs config val {}", val.getRd());
1421             synchronized (BgpConfigurationManager.this) {
1422                 String rd = val.getRd();
1423                 BgpRouter br = getClient(YANG_OBJ);
1424                 if (br == null) {
1425                     LOG.debug("{} Unable to process remove for rd {}; {} {}", YANG_OBJ, rd,
1426                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1427                     return;
1428                 }
1429                 try {
1430                     List<AddressFamiliesVrf> adf = mapNewAdFamily.get(rd);
1431                     adf = adf != null ? adf : new ArrayList<>();
1432                     for (AddressFamiliesVrf s : val.getAddressFamiliesVrf()) {
1433                         br.delVrf(rd, s.getAfi().toJava(), s.getSafi().toJava());
1434                         adf.remove(s);// remove in the map the vrf in waiting for advertise quagga
1435                     }
1436                     if (adf.isEmpty()) {
1437                         mapNewAdFamily.remove(rd);
1438                     }
1439                 } catch (TException | BgpRouterException e) {
1440                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1441                 }
1442             }
1443         }
1444
1445         @Override
1446         protected void update(InstanceIdentifier<Vrfs> iid,
1447                 Vrfs oldval, Vrfs newval) {
1448             if (oldval != null && newval != null) {
1449                 LOG.debug("received update Vrfs config val {}, VRFS: Update getting triggered for VRFS rd {}",
1450                         newval.getRd(), oldval.getRd());
1451             } else {
1452                 LOG.debug("received update Vrfs config val {}, from old vrf {}",
1453                         newval, oldval);
1454             }
1455             if (!isBGPEntityOwner()) {
1456                 return;
1457             }
1458
1459             List<AddressFamiliesVrf> adFamilyVrfToDel = new ArrayList<>();
1460             List<AddressFamiliesVrf> adFamilyVrfToAdd = new ArrayList<>();
1461             List<AddressFamiliesVrf> oldlistAdFamilies = new ArrayList<>();
1462             List<AddressFamiliesVrf> newlistAdFamilies = new ArrayList<>();
1463             if (oldval != null) {
1464                 oldlistAdFamilies = oldval.getAddressFamiliesVrf() == null
1465                         ? new ArrayList<>() : oldval.getAddressFamiliesVrf();
1466             }
1467             if (newval != null) {
1468                 newlistAdFamilies = newval.getAddressFamiliesVrf() == null
1469                         ? new ArrayList<>() : newval.getAddressFamiliesVrf();
1470             }
1471             /*find old AddressFamily to remove from new configuration*/
1472             for (AddressFamiliesVrf adVrf : oldlistAdFamilies) {
1473                 if (!newlistAdFamilies.contains(adVrf)) {
1474                     adFamilyVrfToDel.add(adVrf);
1475                 }
1476             }
1477             /*find new AddressFamily to add to unexisting configuration*/
1478             for (AddressFamiliesVrf adVrf : newlistAdFamilies) {
1479                 if (!oldlistAdFamilies.contains(adVrf)) {
1480                     adFamilyVrfToAdd.add(adVrf);
1481                 }
1482             }
1483             String rd = newval != null ? newval.getRd() : null;
1484             if (rd != null) {
1485                 BgpRouter br = getClient(YANG_OBJ);
1486                 if (br == null) {
1487                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1488                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1489                     return;
1490                 }
1491
1492                 for (AddressFamiliesVrf adfvrf : adFamilyVrfToAdd) {
1493                     try {
1494                         LOG.debug("call addVRf rd {} afi {} safi {}", rd, adfvrf.getAfi(), adfvrf.getSafi());
1495                         br.addVrf(BgpUtil.getLayerType(adfvrf), rd, newval.getImportRts(),
1496                                 newval.getExportRts(),adfvrf.getAfi().toJava(), adfvrf.getSafi().toJava());
1497                     } catch (TException | BgpRouterException e) {
1498                         LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1499                     }
1500                 }
1501
1502                 for (AddressFamiliesVrf adfToDel : adFamilyVrfToDel) {
1503                     try {
1504                         LOG.debug("call delVRf rd {} afi {} safi {}", rd, adfToDel.getAfi(), adfToDel.getSafi());
1505                         br.delVrf(rd, adfToDel.getAfi().toJava(), adfToDel.getSafi().toJava());
1506                     } catch (TException | BgpRouterException e) {
1507                         LOG.error("{} delVrf received exception; {}", YANG_OBJ, ADD_WARN, e);
1508                     }
1509                 }
1510             }
1511         }
1512     }
1513
1514     public class BgpReactor
1515             extends AsyncDataTreeChangeListenerBase<Bgp, BgpReactor>
1516             implements ClusteredDataTreeChangeListener<Bgp> {
1517
1518         private static final String YANG_OBJ = "Bgp ";
1519
1520         public BgpReactor() {
1521             super(Bgp.class, BgpReactor.class);
1522         }
1523
1524
1525         @Override
1526         protected void add(InstanceIdentifier<Bgp> iid, Bgp val) {
1527             LOG.debug("received add Bgp config");
1528
1529             try {
1530                 initer.await();
1531             } catch (InterruptedException e) {
1532                 // Ignored
1533             }
1534             synchronized (BgpConfigurationManager.this) {
1535                 config = val;
1536                 if (!isBGPEntityOwner()) {
1537                     return;
1538                 }
1539             }
1540         }
1541
1542         @Override
1543         protected BgpReactor getDataTreeChangeListener() {
1544             return BgpReactor.this;
1545         }
1546
1547         @Override
1548         protected InstanceIdentifier<Bgp> getWildCardPath() {
1549             return InstanceIdentifier.create(Bgp.class);
1550         }
1551
1552         @Override
1553         protected void remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1554             if (!isBGPEntityOwner()) {
1555                 return;
1556             }
1557             LOG.debug("received remove Bgp config");
1558
1559             config = null;
1560         }
1561
1562         @Override
1563         protected void update(InstanceIdentifier<Bgp> iid,
1564                 Bgp oldval, Bgp newval) {
1565             if (!isBGPEntityOwner()) {
1566                 return;
1567             }
1568
1569             config = newval;
1570         }
1571     }
1572
1573     @SuppressWarnings("deprecation")
1574     public class MultipathReactor
1575             extends AsyncDataTreeChangeListenerBase<Multipath, MultipathReactor>
1576             implements ClusteredDataTreeChangeListener<Multipath> {
1577
1578         private static final String YANG_OBJ = "multipath ";
1579
1580         public MultipathReactor() {
1581             super(Multipath.class, MultipathReactor.class);
1582         }
1583
1584
1585         @Override
1586         protected MultipathReactor getDataTreeChangeListener() {
1587             return MultipathReactor.this;
1588         }
1589
1590         @Override
1591         protected InstanceIdentifier<Multipath> getWildCardPath() {
1592             return InstanceIdentifier.create(Bgp.class).child(MultipathContainer.class).child(Multipath.class);
1593         }
1594
1595         @Override
1596         protected void remove(InstanceIdentifier<Multipath> iid, Multipath val) {
1597             executor.execute(new MultipathStatusChange(val));
1598         }
1599
1600         @Override
1601         protected void update(InstanceIdentifier<Multipath> iid, Multipath oldval, Multipath newval) {
1602             executor.execute(new MultipathStatusChange(newval));
1603         }
1604
1605         @Override
1606         protected void add(InstanceIdentifier<Multipath> key, Multipath dataObjectModification) {
1607             executor.execute(new MultipathStatusChange(dataObjectModification));
1608         }
1609
1610         class MultipathStatusChange implements Runnable {
1611
1612             Multipath multipath;
1613
1614             MultipathStatusChange(Multipath multipath) {
1615                 this.multipath = multipath;
1616             }
1617
1618             @Override
1619             public void run() {
1620                 if (isBGPEntityOwner()) {
1621                     synchronized (BgpConfigurationManager.this) {
1622
1623                         BgpRouter br = getClient(YANG_OBJ);
1624
1625                         if (br != null) {
1626                             af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1627                             af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1628
1629                             try {
1630                                 if (multipath.isMultipathEnabled()) {
1631                                     br.enableMultipath(afi, safi);
1632                                 } else {
1633                                     br.disableMultipath(afi, safi);
1634                                 }
1635                             } catch (TException | BgpRouterException e) {
1636                                 LOG.error("{} received exception", YANG_OBJ, e);
1637                             }
1638                         }
1639                     }
1640                 }
1641             }
1642
1643         }
1644
1645         @Override
1646         public void close() {
1647             super.close();
1648         }
1649     }
1650
1651     @SuppressWarnings("deprecation")
1652     public class VrfMaxpathReactor
1653             extends AsyncDataTreeChangeListenerBase<VrfMaxpath, VrfMaxpathReactor>
1654             implements ClusteredDataTreeChangeListener<VrfMaxpath> {
1655
1656         private static final String YANG_OBJ = "vrfMaxpath ";
1657
1658         public VrfMaxpathReactor() {
1659             super(VrfMaxpath.class, VrfMaxpathReactor.class);
1660         }
1661
1662
1663         @Override
1664         protected VrfMaxpathReactor getDataTreeChangeListener() {
1665             return VrfMaxpathReactor.this;
1666         }
1667
1668         @Override
1669         protected InstanceIdentifier<VrfMaxpath> getWildCardPath() {
1670             return InstanceIdentifier.create(Bgp.class).child(VrfMaxpathContainer.class).child(VrfMaxpath.class);
1671         }
1672
1673         class VrfMaxPathConfigurator implements Runnable {
1674
1675             VrfMaxpath vrfMaxpathVal;
1676
1677             VrfMaxPathConfigurator(VrfMaxpath vrfMaxPathVal) {
1678                 this.vrfMaxpathVal = vrfMaxPathVal;
1679             }
1680
1681             @Override
1682             public void run() {
1683                 if (isBGPEntityOwner()) {
1684                     synchronized (BgpConfigurationManager.this) {
1685                         BgpRouter br = getClient(YANG_OBJ);
1686                         if (br != null) {
1687                             try {
1688                                 br.multipaths(vrfMaxpathVal.getRd(), vrfMaxpathVal.getMaxpaths().toJava());
1689                                 LOG.debug("Maxpath for vrf {} is {}", vrfMaxpathVal.getRd(),
1690                                         vrfMaxpathVal.getMaxpaths());
1691                             } catch (TException | BgpRouterException e) {
1692                                 LOG.error("{} received exception", YANG_OBJ, e);
1693                             }
1694                         }
1695                     }
1696                 }
1697             }
1698         }
1699
1700         @Override
1701         protected void remove(InstanceIdentifier<VrfMaxpath> iid, VrfMaxpath vrfMaxPathVal) {
1702             if (isBGPEntityOwner()) {
1703                 synchronized (BgpConfigurationManager.this) {
1704                     BgpRouter br = getClient(YANG_OBJ);
1705                     if (br != null) {
1706                         try {
1707                             br.multipaths(vrfMaxPathVal.getRd(), BgpConstants.BGP_DEFAULT_MULTIPATH);
1708                             LOG.debug("Del Maxpath for vrf: {} ", vrfMaxPathVal.getRd());
1709                         } catch (TException | BgpRouterException e) {
1710                             LOG.error(YANG_OBJ + " del received exception:", e);
1711                         }
1712                     }
1713                 }
1714             }
1715         }
1716
1717         @Override
1718         protected void update(InstanceIdentifier<VrfMaxpath> iid,
1719                               VrfMaxpath oldval, VrfMaxpath newval) {
1720             if (!Objects.equals(oldval.getMaxpaths(), newval.getMaxpaths())) {
1721                 executor.execute(new VrfMaxPathConfigurator(newval));
1722             }
1723         }
1724
1725         @Override
1726         protected void add(InstanceIdentifier<VrfMaxpath> instanceIdentifier, VrfMaxpath vrfMaxpathVal) {
1727             executor.execute(new VrfMaxPathConfigurator(vrfMaxpathVal));
1728         }
1729
1730         @Override
1731         public void close() {
1732             super.close();
1733         }
1734     }
1735
1736     public class BfdConfigReactor
1737             extends AsyncDataTreeChangeListenerBase<BfdConfig, BfdConfigReactor>
1738             implements ClusteredDataTreeChangeListener<BfdConfig> {
1739
1740         private static final String YANG_OBJ = "BfdConfig ";
1741
1742         public BfdConfigReactor() {
1743             super(BfdConfig.class, BfdConfigReactor.class);
1744         }
1745
1746         @Override
1747         protected void add(InstanceIdentifier<BfdConfig> iid, BfdConfig val) {
1748             if (!isBGPEntityOwner()) {
1749                 return;
1750             }
1751             BgpRouter br = getClient(YANG_OBJ);
1752             LOG.debug("received bfd config: bfd enabled {} min-rx {} min-tx {} detect-mul {} mhop {}",
1753                     val.isBfdEnabled(), val.getMinRx(), val.getMinTx(),
1754                     val.getDetectMult(), val.isMultihop());
1755             if (br == null) {
1756                 LOG.debug(YANG_OBJ + "{} Unable to process add  {}",
1757                         BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1758                 return;
1759             }
1760             if (val.isBfdEnabled() == false) {
1761                 LOG.debug("BFD not enabled. Ignoring the config add");
1762                 return;
1763             }
1764             int minRx = val.getMinRx().intValue();
1765             int minTx = val.getMinTx().intValue();
1766             int detectMult = val.getDetectMult().intValue();
1767             boolean multiHop = val.isMultihop();
1768             try {
1769                 br.addBfd(detectMult, minRx, minTx,multiHop);
1770             } catch (TException | BgpRouterException e) {
1771                 LOG.error("{} get {}, Add received exception;", YANG_OBJ, ADD_WARN, e);
1772             }
1773         }
1774
1775         @Override
1776         protected BfdConfigReactor getDataTreeChangeListener() {
1777             return BfdConfigReactor.this;
1778         }
1779
1780         @Override
1781         protected InstanceIdentifier<BfdConfig> getWildCardPath() {
1782             return InstanceIdentifier.create(BfdConfig.class);
1783         }
1784
1785         @Override
1786         protected void remove(InstanceIdentifier<BfdConfig> iid, BfdConfig val) {
1787             if (!isBGPEntityOwner()) {
1788                 return;
1789             }
1790             LOG.debug("received bfd config removal");
1791             BgpRouter br = getClient(YANG_OBJ);
1792             if (br == null) {
1793                 LOG.debug("{} Unable to process del {}  {}", YANG_OBJ,
1794                         BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1795                 return;
1796             }
1797             try {
1798                 br.delBfd();
1799             } catch (TException | BgpRouterException e) {
1800                 LOG.error("{} get {}, Del received exception;", YANG_OBJ, ADD_WARN, e);
1801             }
1802
1803         }
1804
1805         @Override
1806         protected void update(InstanceIdentifier<BfdConfig> iid,
1807                               BfdConfig oldval, BfdConfig newval) {
1808             LOG.debug("received bfd config: updated oldval bfd enabled {}"
1809                     + "min-rx {} min-tx {} detect-mul {} mhop {}",
1810                     oldval.isBfdEnabled(), oldval.getMinRx(), oldval.getMinTx(),
1811                     oldval.getDetectMult(), oldval.isMultihop());
1812             LOG.debug("received bfd config: updated newval bfd enabled {}"
1813                     + "min-rx {} min-tx {} detect-mul {} mhop {}",
1814                     newval.isBfdEnabled(), newval.getMinRx(), newval.getMinTx(),
1815                     newval.getDetectMult(), newval.isMultihop());
1816             if (oldval.isBfdEnabled()) {
1817                 LOG.debug("deleting bfd config on an update");
1818                 remove(iid, oldval);
1819             }
1820             LOG.debug("adding bfd config on an update");
1821             add(iid, newval);
1822         }
1823     }
1824
1825
1826     public boolean isIpAvailable(String odlip) {
1827
1828         try {
1829             if (odlip != null) {
1830                 if ("127.0.0.1".equals(odlip)) {
1831                     return true;
1832                 }
1833                 Enumeration<NetworkInterface> networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces();
1834                 while (networkInterfaceEnumeration.hasMoreElements()) {
1835                     NetworkInterface networkInterface = networkInterfaceEnumeration.nextElement();
1836                     Enumeration<InetAddress>  inetAddressEnumeration = networkInterface.getInetAddresses();
1837                     while (inetAddressEnumeration.hasMoreElements()) {
1838                         InetAddress inetAddress = inetAddressEnumeration.nextElement();
1839                         if (odlip.equals(inetAddress.getHostAddress())) {
1840                             return true;
1841                         }
1842                     }
1843                 }
1844             }
1845         } catch (SocketException e) {
1846             // Ignored?
1847         }
1848         return false;
1849     }
1850
1851     public long getStalePathtime(int defValue, AsId asId) {
1852         long spt = 0;
1853         try {
1854             InstanceIdentifier<GracefulRestart> id =
1855                     InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
1856             Optional<GracefulRestart> gracefulRestartOptional = MDSALUtil.read(dataBroker,
1857                     LogicalDatastoreType.CONFIGURATION, id);
1858             if (gracefulRestartOptional.isPresent()) {
1859                 spt = gracefulRestartOptional.get().getStalepathTime().toJava();
1860             }
1861         } catch (NullPointerException e) {
1862             try {
1863                 spt = asId.getStalepathTime().toJava();
1864                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1865             } catch (NullPointerException ignore) {
1866                 LOG.trace("BGP AS id is not set using graceful");
1867                 spt = defValue;
1868             }
1869         }
1870         if (spt == 0) {
1871             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1872             spt = defValue;
1873         }
1874         return spt;
1875     }
1876
1877     public static boolean isValidConfigBgpHostPort(String bgpHost, int bgpPort) {
1878         if (!bgpHost.equals(DEF_CHOST)) {
1879             return true;
1880         } else {
1881             return false;
1882         }
1883     }
1884
1885     public synchronized void bgpRestarted() {
1886         /*
1887          * If there a thread which in the process of stale cleanup, cancel it
1888          * and start a new thread (to avoid processing same again).
1889          */
1890         if (previousReplayJobInProgress()) {
1891             cancelPreviousReplayJob();
1892         }
1893         Runnable task = () -> {
1894             try {
1895                 LOG.info("running bgp replay task ");
1896                 if (get() == null) {
1897                     String host = getConfigHost();
1898                     int port = getConfigPort();
1899                     LOG.info("connecting  to bgp host {} ", host);
1900                     bgpRouter.connect(host, port);
1901                     LOG.info("no config to push in bgp replay task ");
1902                     return;
1903                 }
1904                 setStaleStartTime(System.currentTimeMillis());
1905                 LOG.info("started creating stale fibDSWriter  map ");
1906                 createStaleFibMap();
1907                 setStaleEndTime(System.currentTimeMillis());
1908                 LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
1909                 LOG.info("started bgp config replay ");
1910                 setCfgReplayStartTime(System.currentTimeMillis());
1911                 boolean replaySucceded = replay();
1912                 setCfgReplayEndTime(System.currentTimeMillis());
1913                 LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1914                 if (replaySucceded) {
1915                     long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1916                     setStaleCleanupTime(routeSyncTime);
1917                     LOG.error("starting the stale cleanup timer: {} seconds", routeSyncTime);
1918                     routeCleanupFuture = executor.schedule(new RouteCleanup(), routeSyncTime, TimeUnit.SECONDS);
1919                 } else {
1920                     LOG.error("skipping stale cleanup, may be due to exception while replay");
1921                     staledFibEntriesMap.clear();
1922                 }
1923             } catch (InterruptedException | TimeoutException | ExecutionException eCancel) {
1924                 LOG.error("Stale Cleanup Task Cancelled", eCancel);
1925             }
1926         };
1927         lastReplayJobFt = executor.submit(task);
1928     }
1929
1930     private boolean previousReplayJobInProgress() {
1931         return ((lastReplayJobFt != null && !lastReplayJobFt.isDone())
1932                 || (routeCleanupFuture != null && !routeCleanupFuture.isDone()));
1933     }
1934
1935     private void cancelPreviousReplayJob() {
1936         try {
1937             LOG.error("cancelling already running bgp replay task");
1938             if (lastReplayJobFt != null) {
1939                 lastReplayJobFt.cancel(true);
1940                 lastReplayJobFt = null;
1941                 staledFibEntriesMap.clear();
1942             }
1943             if (routeCleanupFuture != null) {
1944                 routeCleanupFuture.cancel(true);
1945                 routeCleanupFuture = null;
1946                 staledFibEntriesMap.clear();
1947             }
1948             Thread.sleep(2000);
1949         } catch (InterruptedException e) {
1950             LOG.error("Failed to cancel previous replay job ", e);
1951         }
1952     }
1953
1954     private void doRouteSync() {
1955         for (af_afi afi : af_afi.values()) {
1956             try {
1957                 bgpRouter.initRibSync(bgpSyncHandle);
1958             } catch (BgpRouterException e) {
1959                 LOG.error("Route sync aborted, exception when initializing", e);
1960                 return;
1961             }
1962             LOG.error("Starting BGP route sync for afi {}", afi.getValue());
1963             while (bgpSyncHandle.getState() != BgpSyncHandle.DONE) {
1964                 Routes routes = null;
1965                 int noUpdates = 0;
1966                 try {
1967                     routes = bgpRouter.doRibSync(bgpSyncHandle, afi);
1968                 } catch (TException | BgpRouterException e) {
1969                     LOG.error("Route sync aborted, exception when syncing", e);
1970                     return;
1971                 }
1972                 Iterator<Update> updates = routes.getUpdatesIterator();
1973                 while (updates.hasNext()) {
1974                     noUpdates++;
1975                     Update update = updates.next();
1976                     String rd = update.getRd();
1977                     String nexthop = update.getNexthop();
1978
1979                     // TODO: decide correct label here
1980                     int label = update.getL3label();
1981                     int l2label = update.getL2label();
1982
1983                     String prefix = update.getPrefix();
1984                     int plen = update.getPrefixlen();
1985
1986
1987                     // TODO: protocol type will not be available in "update"
1988                     // use "rd" to query vrf table and obtain the protocol_type.
1989                     // Currently using PROTOCOL_EVPN as default.
1990                     onUpdatePushRoute(
1991                            protocol_type.PROTOCOL_L3VPN,
1992                            rd,
1993                            prefix,
1994                            plen,
1995                            nexthop,
1996                            update.getMacaddress(),
1997                            Uint32.valueOf(label),
1998                            Uint32.valueOf(l2label),
1999                            update.getRoutermac(),
2000                            afi);
2001                 }
2002                 LOG.error("No of updates for afi {} is {}", afi.getValue(), noUpdates);
2003             }
2004         }
2005         try {
2006             LOG.error("Ending BGP route-sync");
2007             bgpRouter.endRibSync(bgpSyncHandle);
2008         } catch (BgpRouterException e) {
2009             LOG.error("Route sync aborted, exception when ending", e);
2010         }
2011     }
2012
2013     public void addTepToElanDS(String rd, String tepIp, String mac, Uint32 l2vni) {
2014         boolean needUpdate = addToRt2TepMap(rd, tepIp, mac, l2vni);
2015         if (needUpdate) {
2016             LOG.info("Adding tepIp {} with RD {} to ELan DS", tepIp, rd);
2017             bgpUtil.addTepToElanInstance(rd, tepIp);
2018         } else {
2019             LOG.debug("Skipping the Elan update for RT2 from tep {} rd {}", tepIp, rd);
2020         }
2021     }
2022
2023     public void deleteTepfromElanDS(String rd, String tepIp, String mac) {
2024         boolean needUpdate = deleteFromRt2TepMap(rd, tepIp, mac);
2025         if (needUpdate) {
2026             LOG.info("Deleting tepIp {} with RD {} to ELan DS", tepIp, rd);
2027             bgpUtil.deleteTepFromElanInstance(rd, tepIp);
2028         } else {
2029             LOG.debug("Skipping the Elan update for RT2 withdraw from tep {} rd {}", tepIp, rd);
2030         }
2031     }
2032
2033     /* onUpdatePushRoute
2034      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
2035      *  - Entry compare shall include NextHop, Label.
2036      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
2037      *  - If entry not found, add to FIB Config DS.
2038      *  - If entry found, but either Label/NextHop doesn't match.
2039      *      - Update FIB Config DS with modified values.
2040      *      - delete from Stale Map.
2041      */
2042
2043     public void onUpdatePushRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
2044                                   String macaddress, Uint32 label, Uint32 l2label, String routermac, af_afi afi) {
2045         PrefixUpdateEvent prefixUpdateEvent = new PrefixUpdateEvent(protocolType,rd,prefix,plen,nextHop,
2046                 macaddress,label,l2label,routermac,afi);
2047         bgpUpdatesHistory.addToHistory(TransactionType.ADD, prefixUpdateEvent);
2048         boolean addroute = false;
2049         boolean macupdate = false;
2050         Uint32 l3vni = Uint32.ZERO;
2051         VrfEntry.EncapType encapType = VrfEntry.EncapType.Mplsgre;
2052         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
2053             encapType = VrfEntry.EncapType.Vxlan;
2054             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
2055             if (vpnInstanceOpDataEntry != null) {
2056                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
2057                     LOG.info("Got RT2 route for RD {} l3label {} l2label {} from tep {} with mac {} remote RD {}",
2058                             vpnInstanceOpDataEntry.getVpnInstanceName(), label, l2label, nextHop, macaddress, rd);
2059                     addTepToElanDS(rd, nextHop, macaddress, l2label);
2060                     macupdate = true;
2061                 } else {
2062                     l3vni = vpnInstanceOpDataEntry.getL3vni();
2063                 }
2064             } else {
2065                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
2066                 return;
2067             }
2068         }
2069
2070         if (!staledFibEntriesMap.isEmpty()) {
2071             // restart Scenario, as MAP is not empty.
2072             Map<String, Uint32> map = staledFibEntriesMap.get(rd);
2073             if (map != null) {
2074                 String prefixNextHop = appendNextHopToPrefix(prefix + "/" + plen, nextHop);
2075                 Uint32 labelInStaleMap = map.get(prefixNextHop);
2076                 if (null == labelInStaleMap) {
2077                     // New Entry, which happened to be added during restart.
2078                     addroute = true;
2079                 } else {
2080                     map.remove(prefixNextHop);
2081                     if (isRouteModified(label, labelInStaleMap)) {
2082                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
2083                         // Existing entry, where in Label got modified during restart
2084                         addroute = true;
2085                     }
2086                 }
2087             } else {
2088                 LOG.debug("rd {} map is null while processing prefix {} ", rd, prefix);
2089                 addroute = true;
2090             }
2091         } else {
2092             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
2093             addroute = true;
2094         }
2095         if (macupdate) {
2096             LOG.info("ADD: Adding Mac Fib entry rd {} mac{} nexthop {} l2vni {}", rd, macaddress, nextHop, l2label);
2097             fibDSWriter.addMacEntryToDS(rd, macaddress, prefix, Collections.singletonList(nextHop),
2098                     encapType, l2label, routermac, RouteOrigin.BGP);
2099             LOG.info("ADD: Added Mac Fib entry rd {} prefix {} nexthop {} label {}", rd, macaddress, nextHop, l2label);
2100         } else if (addroute) {
2101             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {} afi {}",
2102                     rd, prefix, nextHop, label, afi);
2103             // TODO: modify addFibEntryToDS signature
2104             List<String> nextHopList = Collections.singletonList(nextHop);
2105             fibDSWriter.addFibEntryToDS(rd, prefix + "/" + plen, nextHopList, encapType, label, l3vni,
2106                                         routermac, RouteOrigin.BGP);
2107             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
2108             String vpnName = bgpUtil.getVpnNameFromRd(rd);
2109             if (vpnName != null) {
2110                 vpnLinkService.leakRouteIfNeeded(vpnName, prefix, nextHopList, label, RouteOrigin.BGP,
2111                                                  NwConstants.ADD_FLOW);
2112             }
2113         }
2114     }
2115
2116     public void onUpdateWithdrawRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
2117             String macaddress) {
2118         PrefixWithdrawEvent prefixWithdrawEvent = new PrefixWithdrawEvent(protocolType,rd,prefix,plen,
2119                 nextHop,macaddress);
2120         bgpUpdatesHistory.addToHistory(TransactionType.ADD, prefixWithdrawEvent);
2121         boolean macupdate = false;
2122         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
2123             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
2124             if (vpnInstanceOpDataEntry != null) {
2125                 Uint32 vni = vpnInstanceOpDataEntry.getL3vni();
2126                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
2127                     LOG.debug("Got RT2 withdraw for RD {} {} from tep {} with mac {} remote RD {}",
2128                             vpnInstanceOpDataEntry.getVpnInstanceName(), vni, nextHop, macaddress, rd);
2129                     deleteTepfromElanDS(rd, nextHop, macaddress);
2130                     LOG.debug("For rd {}. skipping fib update", rd);
2131                     macupdate = true;
2132                 }
2133             } else {
2134                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
2135                 return;
2136             }
2137         }
2138         if (macupdate) {
2139             LOG.info("Removing Mac Fib entry rd {} mac{} nexthop {} ", rd, macaddress, nextHop);
2140             fibDSWriter.removeMacEntryFromDS(rd, macaddress);
2141             LOG.info("Removed Mac Fib entry rd {} prefix {} nexthop {} ", rd, macaddress, nextHop);
2142         } else {
2143             LOG.info("REMOVE: Removing Fib entry rd {} prefix {}", rd, prefix);
2144             fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nextHop);
2145             LOG.info("REMOVE: Removed Fib entry rd {} prefix {}", rd, prefix);
2146         }
2147     }
2148
2149     //TODO: below function is for testing purpose with cli
2150     public void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
2151         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
2152         fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nexthop);
2153         String vpnName = bgpUtil.getVpnNameFromRd(rd);
2154         if (vpnName != null) {
2155             vpnLinkService.leakRouteIfNeeded(vpnName, prefix, null /*nextHopList*/, Uint32.ZERO /*INVALID_LABEL*/,
2156                                              RouteOrigin.BGP, NwConstants.DEL_FLOW);
2157         }
2158     }
2159
2160     public void peerDown(String ipAddress, long asNumber) {
2161         PeerDownEvent peerDownEvent = new PeerDownEvent(ipAddress,asNumber);
2162         bgpUpdatesHistory.addToHistory(TransactionType.ADD, peerDownEvent);
2163         List<String> tepIpList = bgpUtil.getDcgwTepConfig(ipAddress);
2164         if (tepIpList == null) {
2165             LOG.error("No Tep IP configured for DCGW {} on a peerDown", ipAddress);
2166             return;
2167         }
2168         tepIpList.forEach(tepIp -> {
2169             bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW);
2170         });
2171     }
2172
2173     public void peerUp(String ipAddress, long asNumber) {
2174         PeerUpEvent peerUpEvent = new PeerUpEvent(ipAddress,asNumber);
2175         bgpUpdatesHistory.addToHistory(TransactionType.ADD, peerUpEvent);
2176         List<String> tepIpList = bgpUtil.getDcgwTepConfig(ipAddress);
2177         if (tepIpList == null) {
2178             LOG.error("No Tep IP configured for DCGW {} on a peerUp", ipAddress);
2179             return;
2180         }
2181         tepIpList.forEach(tepIp -> {
2182             bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW);
2183         });
2184     }
2185
2186     private static boolean isRouteModified(Uint32 label, Uint32 labelInStaleMap) {
2187         return labelInStaleMap != null && !labelInStaleMap.equals(label);
2188     }
2189
2190     static class ReplayNbr {
2191         Neighbors nbr;
2192         boolean shouldRetry = false;
2193
2194         public Neighbors getNbr() {
2195             return nbr;
2196         }
2197
2198         public boolean isShouldRetry() {
2199             return shouldRetry;
2200         }
2201
2202         public void setShouldRetry(boolean retryNbr) {
2203             this.shouldRetry = retryNbr;
2204         }
2205
2206         ReplayNbr(Neighbors nbr, boolean shouldRetry) {
2207             this.nbr = nbr;
2208             this.shouldRetry = shouldRetry;
2209         }
2210     }
2211
2212     private static boolean replayNbrConfig(List<Neighbors> neighbors, BgpRouter br) {
2213         if (neighbors == null || neighbors.isEmpty()) {
2214             LOG.error("Replaying nbr configuration, received NULL list ");
2215             return true;
2216         }
2217
2218         List<ReplayNbr> replayNbrList = new ArrayList<>();
2219         for (Neighbors nbr : neighbors) {
2220             if (nbr != null) {
2221                 replayNbrList.add(new ReplayNbr(nbr, true));
2222             }
2223         }
2224         final int numberOfNbrRetries = 3;
2225         RetryOnException nbrRetry = new RetryOnException(numberOfNbrRetries);
2226         do {
2227             for (ReplayNbr replayNbr : replayNbrList) {
2228                 if (!replayNbr.isShouldRetry()) {
2229                     continue;
2230                 }
2231                 boolean replayDone = false;
2232                 LOG.debug("Replaying addNbr {}", replayNbr.getNbr().getAddress().getValue());
2233                 replayDone = false;
2234                 try {
2235                     final String md5password = extractMd5Secret(replayNbr.getNbr());
2236                     br.addNeighbor(replayNbr.getNbr().getAddress().getValue(),
2237                             replayNbr.getNbr().getRemoteAs().longValue(), md5password);
2238                     replayDone = true;
2239                 } catch (TApplicationException tae) {
2240                     LOG.debug("Replaying addNbr {}, tapplicationexception: ",
2241                             replayNbr.getNbr().getAddress().getValue(), tae);
2242                     if (tae.getType() == BgpRouterException.BGP_ERR_PEER_EXISTS) {
2243                         LOG.debug("Replaying addNbr Neighbor already present");
2244                         replayDone = true;
2245                     } else {
2246                         LOG.error("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), tae);
2247                     }
2248                 } catch (TException | BgpRouterException eNbr) {
2249                     LOG.debug("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), eNbr);
2250                 }
2251
2252                 LOG.debug("Replay addNbr {} successful", replayNbr.getNbr().getAddress().getValue());
2253
2254                 //Update Source handling
2255                 UpdateSource us = replayNbr.getNbr().getUpdateSource();
2256                 if (us != null) {
2257                     LOG.debug("Replaying updatesource {} to peer {}", us.getSourceIp().getValue(),
2258                             us.getPeerIp().getValue());
2259                     try {
2260                         br.addUpdateSource(us.getPeerIp().getValue(),
2261                                 us.getSourceIp().getValue());
2262                     } catch (TException | BgpRouterException eUs) {
2263                         LOG.debug("Replaying UpdateSource for Nbr {}, exception:",
2264                                 replayNbr.getNbr().getAddress().getValue(), eUs);
2265                     }
2266                     LOG.debug("Replay updatesource {} successful", us.getSourceIp().getValue());
2267                 }
2268                 //Ebgp Multihope
2269                 EbgpMultihop en = replayNbr.getNbr().getEbgpMultihop();
2270                 if (en != null) {
2271                     try {
2272                         br.addEbgpMultihop(en.getPeerIp().getValue(),
2273                                 en.getNhops().intValue());
2274                     } catch (TException | BgpRouterException eEbgpMhop) {
2275                         LOG.debug("Replaying EbgpMultihop for Nbr {}, exception: ",
2276                                 replayNbr.getNbr().getAddress().getValue(), eEbgpMhop);
2277                     }
2278                 }
2279
2280                 //afs
2281                 List<AddressFamilies> afs = replayNbr.getNbr().getAddressFamilies();
2282                 if (afs != null) {
2283                     for (AddressFamilies af : afs) {
2284                         af_afi afi = af_afi.findByValue(af.getAfi().intValue());
2285                         af_safi safi = af_safi.findByValue(af.getSafi().intValue());
2286                         try {
2287                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
2288                         } catch (TException | BgpRouterException eAFs) {
2289                             LOG.debug("Replaying AddressFamily for Nbr {}, exception:",
2290                                     replayNbr.getNbr().getAddress().getValue(), eAFs);
2291                         }
2292                     }
2293                 }
2294                 //replay is success --> no need to replay this nbr in next iteration.
2295                 replayNbr.setShouldRetry(replayDone ? false : true);
2296             }
2297         } while (nbrRetry.decrementAndRetry());
2298         boolean replaySuccess = true;
2299         for (ReplayNbr replayNbr : replayNbrList) {
2300             replaySuccess = replaySuccess && !replayNbr.isShouldRetry();
2301             if (replaySuccess == false) {
2302                 LOG.error("replayNbrConfig: will be cancelling stale cleanup, cfg nbr: {} Failed:",
2303                         replayNbr.getNbr().getAddress().getValue());
2304             }
2305         }
2306         return replaySuccess;
2307     }
2308
2309     public String getConfigHost() {
2310         if (config == null) {
2311             return hostStartup;
2312         }
2313         ConfigServer ts = config.getConfigServer();
2314         return ts == null ? hostStartup : ts.getHost().getValue();
2315     }
2316
2317     public int getConfigPort() {
2318         if (config == null) {
2319             return Integer.parseInt(portStartup);
2320         }
2321         ConfigServer ts = config.getConfigServer();
2322         return ts == null ? Integer.parseInt(portStartup) :
2323                 ts.getPort().intValue();
2324     }
2325
2326     public Bgp getConfig() {
2327         AtomicInteger bgpDSretryCount = new AtomicInteger(DS_RETRY_COUNT);
2328         while (0 != bgpDSretryCount.decrementAndGet()) {
2329             try {
2330                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
2331                         InstanceIdentifier.create(Bgp.class)).orNull();
2332             } catch (ReadFailedException e) {
2333                 //Config DS may not be up, so sleep for 1 second and retry
2334                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
2335                 try {
2336                     Thread.sleep(WAIT_TIME_BETWEEN_EACH_TRY_MILLIS);
2337                 } catch (InterruptedException timerEx) {
2338                     LOG.debug("WAIT_TIME_BETWEEN_EACH_TRY_MILLIS, Timer got interrupted while waiting for"
2339                             + "config DS availability", timerEx);
2340                 }
2341             }
2342         }
2343         LOG.error("failed to get bgp config");
2344         return null;
2345     }
2346
2347     @SuppressWarnings("checkstyle:IllegalCatch")
2348     public synchronized boolean replay() throws InterruptedException, TimeoutException, ExecutionException {
2349         boolean replaySucceded = true;
2350         boolean doRouteSync = false;
2351         String host = getConfigHost();
2352         int port = getConfigPort();
2353         LOG.error("connecting  to bgp host {} ", host);
2354         boolean res = bgpRouter.connect(host, port);
2355         if (!res) {
2356             LOG.error("Cannot connect to BGP config server at {} {}", host, port);
2357             return replaySucceded;
2358         }
2359         config = getConfig();
2360         if (config == null) {
2361             LOG.error("bgp config is empty nothing to push to bgp");
2362             return replaySucceded;
2363         }
2364         BgpRouter br = bgpRouter;
2365         AsId asId = config.getAsId();
2366         if (asId == null) {
2367             LOG.error("bgp as-id is null");
2368             return replaySucceded;
2369         }
2370         long asNum = asId.getLocalAs().toJava();
2371         IpAddress routerId = asId.getRouterId();
2372         String rid = routerId == null ? "" : routerId.stringValue();
2373         int stalepathTime = (int) getStalePathtime(bgpGrRestartTime, config.getAsId());
2374         boolean announceFbit = true;
2375         boolean replayDone = false;
2376         final int numberOfStartBgpRetries = 3;
2377         RetryOnException startBgpRetry = new RetryOnException(numberOfStartBgpRetries);
2378         do {
2379             try {
2380                 LOG.debug("Replaying BGPConfig ");
2381                 br.startBgp(asNum, rid, bgpKaTime, bgpHoldTime, stalepathTime, announceFbit);
2382                 LOG.debug("Replay BGPConfig successful");
2383                 replayDone = true;
2384                 break;
2385             } catch (BgpRouterException bre) {
2386                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
2387                     LOG.debug("Starting the routesync for exception", bre);
2388                     startBgpRetry.errorOccured();
2389                     if (!startBgpRetry.shouldRetry()) {
2390                         doRouteSync = true;
2391                         replayDone = true;
2392                     }
2393                 } else {
2394                     LOG.error("Replay: startBgp() received exception error {} : ",
2395                             bre.getErrorCode(), bre);
2396                     startBgpRetry.errorOccured();
2397                 }
2398             } catch (TApplicationException tae) {
2399                 if (tae.getType() == BgpRouterException.BGP_ERR_ACTIVE) {
2400                     LOG.debug("Starting the routesync for exception", tae);
2401                     startBgpRetry.errorOccured();
2402                     if (!startBgpRetry.shouldRetry()) {
2403                         doRouteSync = true;
2404                         replayDone = true;
2405                     }
2406                 } else if (tae.getType() == BgpRouterException.BGP_ERR_COMMON_FAILURE) {
2407                     LOG.debug("Starting the routesync for AS-ID started exception", tae);
2408                     startBgpRetry.errorOccured();
2409                     if (!startBgpRetry.shouldRetry()) {
2410                         doRouteSync = true;
2411                         replayDone = true;
2412                     }
2413                 } else {
2414                     LOG.error("Replay: startBgp() received exception type {}: ",
2415                             tae.getType(), tae);
2416                     startBgpRetry.errorOccured();
2417                 }
2418             } catch (Exception e) {
2419                 //not unusual. We may have restarted & BGP is already on
2420                 LOG.error("Replay:startBgp() received exception: ", e);
2421                 startBgpRetry.errorOccured();
2422             }
2423         } while (startBgpRetry.shouldRetry());
2424
2425         replaySucceded = replayDone;
2426
2427         startBgpCountersTask();
2428         startBgpAlarmsTask();
2429
2430         /*
2431          * commenting this due to a bug with QBGP. Will uncomment once QBGP fix is done.
2432          * This wont have any functional impacts
2433          */
2434         //try {
2435         //    br.delayEOR(delayEorSeconds);
2436         //} catch (TException | BgpRouterException e) {
2437         //    LOG.error("Replay: delayEOR() number of seconds to wait for EOR from ODL:", e);
2438         //}
2439
2440         BfdConfig bfdConfig = bgpUtil.getBfdConfig();
2441         if (bfdConfig != null) {
2442             if (bfdConfig.isBfdEnabled()) {
2443                 LOG.debug("Replaying bfd config min-rx {} min-tx {} detect-mul {} mhop {}",
2444                         bfdConfig.getMinRx(), bfdConfig.getMinTx(),
2445                         bfdConfig.getDetectMult(), bfdConfig.isMultihop());
2446                 try {
2447                     br.addBfd(bfdConfig.getDetectMult().intValue(), bfdConfig.getMinRx().intValue(),
2448                             bfdConfig.getMinTx().intValue(), bfdConfig.isMultihop());
2449                 } catch (TApplicationException tae) {
2450                     if (tae.getType() == BgpRouterException.BGP_ERR_PEER_EXISTS) {
2451                         LOG.debug("Replay:addBfd() received exception", tae);
2452                     } else {
2453                         LOG.error("Replay:addBfd() received exception", tae);
2454                     }
2455                 } catch (TException | BgpRouterException e) {
2456                     LOG.error("Replay:addBfd() received exception", e);
2457                 }
2458             }
2459         }
2460
2461         List<Neighbors> neighbors = config.getNeighborsContainer() == null ? null
2462                 : config.getNeighborsContainer().getNeighbors();
2463         if (neighbors != null) {
2464             LOG.error("configuring existing Neighbors present for replay total neighbors {}", neighbors.size());
2465             boolean neighborConfigReplayResult = replayNbrConfig(neighbors, br);
2466             if (neighborConfigReplayResult == false) {
2467                 replaySucceded = false;
2468             }
2469         } else {
2470             LOG.error("no Neighbors present for replay config ");
2471         }
2472
2473         Logging logging = config.getLogging();
2474         if (logging != null) {
2475             try {
2476                 br.setLogging(logging.getFile(), logging.getLevel());
2477             } catch (TException | BgpRouterException e) {
2478                 LOG.error("Replay:setLogging() received exception", e);
2479             }
2480         }
2481
2482         GracefulRestart gracefulRestart = config.getGracefulRestart();
2483         bgpGrRestartTime = ((gracefulRestart != null)
2484                 ? gracefulRestart.getStalepathTime().intValue() : bgpGrRestartTime);
2485         try {
2486             br.addGracefulRestart(bgpGrRestartTime);
2487         } catch (Exception e) {
2488             LOG.error("Replay:addGr() received exception: ", e);
2489         }
2490         List<Vrfs> vrfs = config.getVrfsContainer() == null ? null
2491                 : config.getVrfsContainer().getVrfs();
2492         if (vrfs == null) {
2493             vrfs = new ArrayList<>();
2494         }
2495         for (Vrfs vrf : vrfs) {
2496             for (AddressFamiliesVrf adf : vrf.getAddressFamiliesVrf()) {
2497                 try {
2498                     br.addVrf(BgpUtil.getLayerType(adf), vrf.getRd(), vrf.getImportRts(),
2499                             vrf.getExportRts(), adf.getAfi().toJava(), adf.getSafi().toJava());
2500                 } catch (TException | BgpRouterException e) {
2501                     LOG.error("Replay:addVrf() received exception", e);
2502                 }
2503             }
2504         }
2505
2506
2507         List<Networks> ln = config.getNetworksContainer() == null ? null
2508                 : config.getNetworksContainer().getNetworks();
2509         if (ln != null) {
2510             for (Networks net : ln) {
2511                 String rd = net.getRd();
2512                 String pfxlen = net.getPrefixLen();
2513                 String nh = net.getNexthop().getValue();
2514                 Long label = net.getLabel().toJava();
2515                 int lbl = label == null ? 0 : label.intValue();
2516                 int l3vni = net.getL3vni() == null ? 0 : net.getL3vni().intValue();
2517                 int l2vni = net.getL2vni() == null ? 0 : net.getL2vni().intValue();
2518                 if (rd == null && lbl > 0) {
2519                     //LU prefix is being deleted.
2520                     rd = Integer.toString(lbl);
2521                 }
2522
2523                 BgpControlPlaneType protocolType = net.getBgpControlPlaneType();
2524                 int ethernetTag = net.getEthtag().intValue();
2525                 String esi = net.getEsi();
2526                 String macaddress = net.getMacaddress();
2527                 EncapType encapType = net.getEncapType();
2528                 String routerMac = net.getRoutermac();
2529
2530                 try {
2531                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni,
2532                             BgpUtil.convertToThriftProtocolType(protocolType),
2533                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
2534                 } catch (TException | BgpRouterException e) {
2535                     LOG.error("Replay:addPfx() received exception", e);
2536                 }
2537             }
2538         }
2539
2540
2541         List<Multipath> multipaths = config.getMultipathContainer() == null ? null
2542                 : config.getMultipathContainer().getMultipath();
2543
2544         if (multipaths != null) {
2545             for (Multipath multipath : multipaths) {
2546                 if (multipath != null) {
2547                     af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
2548                     af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
2549
2550                     try {
2551                         if (multipath.isMultipathEnabled()) {
2552                             br.enableMultipath(afi, safi);
2553                         } else {
2554                             br.disableMultipath(afi, safi);
2555                         }
2556                     } catch (TException | BgpRouterException e) {
2557                         LOG.info("Replay:multipaths() received exception", e);
2558                     }
2559                 }
2560             }
2561         }
2562         List<VrfMaxpath> vrfMaxpaths = config.getVrfMaxpathContainer() == null ? null
2563                 : config.getVrfMaxpathContainer().getVrfMaxpath();
2564         if (vrfMaxpaths != null) {
2565             for (VrfMaxpath vrfMaxpath : vrfMaxpaths) {
2566                 try {
2567                     br.multipaths(vrfMaxpath.getRd(), vrfMaxpath.getMaxpaths().toJava());
2568                 } catch (TException | BgpRouterException e) {
2569                     LOG.info("Replay:vrfMaxPath() received exception", e);
2570                 }
2571             }
2572         }
2573
2574         //send End of Rib Marker to Qthriftd.
2575         final int numberOfEORRetries = 3;
2576         RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
2577         do {
2578             try {
2579                 br.sendEOR();
2580                 LOG.debug("Replay sendEOR() successful");
2581                 break;
2582             } catch (Exception e) {
2583                 eorRetry.errorOccured();
2584                 LOG.error("Replay:sedEOR() received exception:", e);
2585             }
2586         } while (eorRetry.shouldRetry());
2587
2588         if (doRouteSync) {
2589             LOG.debug("starting route sync for Thrift BGP_ERR_COMMON_FAILURE exception "
2590                       + "happened earlier");
2591             doRouteSync();
2592         }
2593
2594         return replaySucceded;
2595     }
2596
2597     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
2598         bgpUtil.update(iid, dto);
2599     }
2600
2601     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
2602         bgpUtil.delete(iid);
2603     }
2604
2605     public void startConfig(String bgpHost, int thriftPort) {
2606         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2607                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2608         InstanceIdentifier<ConfigServer> iid = iib.build();
2609         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
2610         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
2611                 .setPort((long) thriftPort).build();
2612         update(iid, dto);
2613     }
2614
2615     public void startBgp(long as, String routerId, int spt, boolean fbit) {
2616         IpAddress rid = routerId == null ? null : IpAddressBuilder.getDefaultInstance(routerId);
2617         Long staleTime = (long) spt;
2618         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2619                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2620         InstanceIdentifier<AsId> iid = iib.build();
2621         AsId dto = new AsIdBuilder().setLocalAs(as)
2622                 .setRouterId(rid)
2623                 .setStalepathTime(staleTime)
2624                 .setAnnounceFbit(fbit).build();
2625         update(iid, dto);
2626     }
2627
2628     public void startBfd(long detectMult, long minRx, long minTx, boolean multiHop) {
2629         InstanceIdentifier.InstanceIdentifierBuilder<BfdConfig> iib =
2630                 InstanceIdentifier.builder(BfdConfig.class);
2631         InstanceIdentifier<BfdConfig> iid = iib.build();
2632         BfdConfig dto = new BfdConfigBuilder()
2633                 .setBfdEnabled(true)
2634                 .setMultihop(multiHop)
2635                 .setMinRx(minRx)
2636                 .setMinTx(minTx)
2637                 .setDetectMult(detectMult)
2638                 .build();
2639         update(iid, dto);
2640     }
2641
2642     public void addDcgwTep(String dcgwIp, String tepIp) {
2643         InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2644                 InstanceIdentifier.builder(Bgp.class)
2645                         .child(DcgwTepList.class)
2646                         .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2647         InstanceIdentifier<DcgwTep> iid = iib.build();
2648         ArrayList<String> tepList = new ArrayList<String>();
2649         tepList.add(tepIp);
2650         DcgwTep dto = new DcgwTepBuilder().setDcGwIp(dcgwIp).setTepIps(tepList)
2651                 .build();
2652         update(iid, dto);
2653         bgpUtil.removeOrUpdateLBGroups(tepIp,NwConstants.MOD_FLOW);
2654     }
2655
2656     public void addLogging(String fileName, String logLevel) {
2657         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2658                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2659         InstanceIdentifier<Logging> iid = iib.build();
2660         Logging dto = new LoggingBuilder().setFile(fileName)
2661                 .setLevel(logLevel).build();
2662         update(iid, dto);
2663     }
2664
2665     public void addGracefulRestart(int staleTime) {
2666         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2667                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
2668         InstanceIdentifier<GracefulRestart> iid = iib.build();
2669         GracefulRestart dto = new GracefulRestartBuilder()
2670                 .setStalepathTime((long) staleTime).build();
2671         update(iid, dto);
2672     }
2673
2674     public void addNeighbor(
2675             String nbrIp, long remoteAs, @Nullable final TcpMd5SignaturePasswordType md5Secret) {
2676         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2677         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2678                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2679                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2680         InstanceIdentifier<Neighbors> iid = iib.build();
2681         TcpSecurityOption tcpSecOption = null;
2682         if (md5Secret != null) {
2683             tcpSecOption = new TcpMd5SignatureOptionBuilder().setTcpMd5SignaturePassword(md5Secret).build();
2684         } // else let tcpSecOption be null
2685         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
2686                 .setRemoteAs(remoteAs).setTcpSecurityOption(tcpSecOption).build();
2687         update(iid, dto);
2688     } // public addNeighbor(nbrIp, remoteAs, md5Secret)
2689
2690     public void addUpdateSource(String nbrIp, String srcIp) {
2691         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2692         Ipv4Address srcAddr = new Ipv4Address(srcIp);
2693         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2694                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2695                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2696                         .child(UpdateSource.class);
2697         InstanceIdentifier<UpdateSource> iid = iib.build();
2698         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
2699                 .setSourceIp(srcAddr).build();
2700         update(iid, dto);
2701     }
2702
2703     public void addEbgpMultihop(String nbrIp, int hops) {
2704         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2705         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2706                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2707                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2708                         .child(EbgpMultihop.class);
2709         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2710         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
2711                 .setNhops((long) hops).build();
2712         update(iid, dto);
2713     }
2714
2715     public void addAddressFamily(String nbrIp, int afi, int safi) {
2716         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2717         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2718                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2719                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2720                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2721         InstanceIdentifier<AddressFamilies> iid = iib.build();
2722         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
2723                 .setAfi((long) afi).setSafi((long) safi).build();
2724         update(iid, dto);
2725     }
2726
2727     public void addPrefix(String rd, String macAddress, String pfx, List<String> nhList,
2728               VrfEntry.EncapType encapType, Uint32 lbl, Uint32 l3vni, Uint32 l2vni, String gatewayMac) {
2729         for (String nh : nhList) {
2730             Ipv4Address nexthop = nh != null ? new Ipv4Address(nh) : null;
2731             Uint32 label = lbl;
2732             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
2733                     .child(NetworksContainer.class)
2734                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
2735             NetworksBuilder networksBuilder = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
2736                                                 .setLabel(label).setEthtag(BgpConstants.DEFAULT_ETH_TAG);
2737             buildVpnEncapSpecificInfo(networksBuilder, encapType, label, l3vni, l2vni, macAddress, gatewayMac);
2738             update(iid, networksBuilder.build());
2739         }
2740     }
2741
2742     private static void buildVpnEncapSpecificInfo(NetworksBuilder builder, VrfEntry.EncapType encapType, Uint32 label,
2743                                                   Uint32 l3vni, Uint32 l2vni, String macAddress, String gatewayMac) {
2744         if (encapType.equals(VrfEntry.EncapType.Mplsgre)) {
2745             builder.setLabel(label).setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLL3VPN)
2746                     .setEncapType(EncapType.GRE);
2747         } else {
2748             builder.setL3vni(l3vni).setL2vni(l2vni).setMacaddress(macAddress).setRoutermac(gatewayMac)
2749                     .setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLEVPN).setEncapType(EncapType.VXLAN);
2750         }
2751     }
2752
2753     // TODO: add LayerType as arg - supports command
2754     public void addVrf(String rd, List<String> irts, List<String> erts, AddressFamily addressFamily) {
2755         Vrfs vrf = bgpUtil.getVrfFromRd(rd);
2756         List<AddressFamiliesVrf> adfList = new ArrayList<>(1);
2757         if (vrf != null) {
2758             adfList = vrf.getAddressFamiliesVrf();
2759         }
2760         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2761         if (addressFamily.equals(AddressFamily.IPV4)) {
2762             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2763             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2764         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2765             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2766             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2767         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2768             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2769             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2770         }
2771         AddressFamiliesVrf adf = adfBuilder.build();
2772         adfList.add(adf);
2773         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib = InstanceIdentifier.builder(Bgp.class)
2774                 .child(VrfsContainer.class)
2775                 .child(Vrfs.class, new VrfsKey(rd));
2776         InstanceIdentifier<Vrfs> iid = iib.build();
2777         Vrfs dto = new VrfsBuilder().setRd(rd).setImportRts(irts)
2778             .setExportRts(erts).setAddressFamiliesVrf(adfList).build();
2779
2780         List<AddressFamiliesVrf> listAdFamilies = mapNewAdFamily.get(rd);
2781         if (listAdFamilies != null) {
2782             listAdFamilies.add(adf);
2783         } else {
2784             mapNewAdFamily.put(rd, adfList);
2785         }
2786
2787         try {
2788             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
2789         } catch (TransactionCommitFailedException e) {
2790             LOG.error("Error adding VRF to datastore", e);
2791             throw new RuntimeException(e);
2792         }
2793
2794         // enable multipath by default in all VRFs
2795         setMultipaths(rd, BgpConstants.BGP_DEFAULT_MULTIPATH);
2796     }
2797
2798     public void stopConfig() {
2799         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2800                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2801         InstanceIdentifier<ConfigServer> iid = iib.build();
2802         delete(iid);
2803     }
2804
2805     public void stopBgp() {
2806         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2807                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2808         InstanceIdentifier<AsId> iid = iib.build();
2809         delete(iid);
2810     }
2811
2812     public void stopBfd() {
2813         InstanceIdentifier.InstanceIdentifierBuilder<BfdConfig> iib =
2814                 InstanceIdentifier.builder(BfdConfig.class);
2815         InstanceIdentifier<BfdConfig> iid = iib.build();
2816         delete(iid);
2817     }
2818
2819     public void delDcgwTep(String dcgwIp, String tepIp) {
2820         if (tepIp == null) {
2821             InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2822                     InstanceIdentifier.builder(Bgp.class)
2823                             .child(DcgwTepList.class)
2824                             .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2825             InstanceIdentifier<DcgwTep> iid = iib.build();
2826             delete(iid);
2827         } else {
2828             InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2829                     InstanceIdentifier.builder(Bgp.class)
2830                             .child(DcgwTepList.class)
2831                             .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2832             InstanceIdentifier<DcgwTep> iid = iib.build();
2833             List<String> tepIpList = bgpUtil.getDcgwTepConfig(dcgwIp);
2834             if (tepIpList == null) {
2835                 LOG.error("No Tep IP configured for DCGW {} on deleting the dcgwtep", dcgwIp);
2836                 return;
2837             }
2838             List<String> newTepIpList = new ArrayList<String>();
2839             tepIpList.forEach(tep -> {
2840                 if (!tep.equals(tepIp)) {
2841                     newTepIpList.add(tep);
2842                 }
2843             });
2844             DcgwTep dto = new DcgwTepBuilder().setDcGwIp(dcgwIp).setTepIps(newTepIpList)
2845                     .build();
2846             try {
2847                 SingleTransactionDataBroker.syncWrite(dataBroker,
2848                         LogicalDatastoreType.CONFIGURATION, iid, dto);
2849             } catch (TransactionCommitFailedException e) {
2850                 LOG.error("delDcgwTep: Error deleting DCGW Tep", e);
2851                 throw new RuntimeException(e);
2852             }
2853         }
2854     }
2855
2856     public void delLogging() {
2857         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2858                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2859         InstanceIdentifier<Logging> iid = iib.build();
2860         delete(iid);
2861     }
2862
2863     public void delGracefulRestart() {
2864         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2865                 InstanceIdentifier.builder(Bgp.class)
2866                         .child(GracefulRestart.class);
2867         InstanceIdentifier<GracefulRestart> iid = iib.build();
2868         delete(iid);
2869     }
2870
2871     public void delNeighbor(String nbrIp) {
2872         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2873         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2874                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2875                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2876         InstanceIdentifier<Neighbors> iid = iib.build();
2877         delete(iid);
2878     }
2879
2880     public void delUpdateSource(String nbrIp) {
2881         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2882         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2883                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2884                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2885                         .child(UpdateSource.class);
2886         InstanceIdentifier<UpdateSource> iid = iib.build();
2887         delete(iid);
2888     }
2889
2890     public void delEbgpMultihop(String nbrIp) {
2891         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2892         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2893                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2894                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2895                         .child(EbgpMultihop.class);
2896         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2897         delete(iid);
2898     }
2899
2900     public void delAddressFamily(String nbrIp, int afi, int safi) {
2901         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2902         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2903                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2904                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2905                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2906         InstanceIdentifier<AddressFamilies> iid = iib.build();
2907         delete(iid);
2908     }
2909
2910     public void delPrefix(String rd, String pfx) {
2911         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
2912                 InstanceIdentifier.builder(Bgp.class).child(NetworksContainer.class)
2913                         .child(Networks.class, new NetworksKey(pfx, rd));
2914         InstanceIdentifier<Networks> iid = iib.build();
2915         delete(iid);
2916     }
2917
2918     public boolean delVrf(String rd, AddressFamily addressFamily) {
2919         if (addressFamily == null) {
2920             LOG.error("delVrf: vrf {}, addressFamily invalid", rd);
2921             return false;
2922         }
2923         delMultipaths(rd);
2924         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2925         if (addressFamily.equals(AddressFamily.IPV4)) {
2926             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2927             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2928         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2929             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2930             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2931         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2932             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2933             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2934         }
2935         LOG.debug("delVrf: Received Delete VRF : rd:{}, address family: {} {}", rd,
2936                 adfBuilder.getAfi(), adfBuilder.getSafi());
2937
2938         Vrfs vrfOriginal = bgpUtil.getVrfFromRd(rd);
2939         if (vrfOriginal == null) {
2940             LOG.error("delVrf: no vrf with existing rd {}. step aborted", rd);
2941             return false;
2942         }
2943
2944         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2945                 InstanceIdentifier.builder(Bgp.class).child(VrfsContainer.class)
2946                         .child(Vrfs.class, new VrfsKey(rd));
2947
2948         InstanceIdentifier<Vrfs> iid = iib.build();
2949
2950         @SuppressWarnings("static-access")
2951         InstanceIdentifier<Bgp> iid6 =  iid.builder(Bgp.class).build()
2952                 .child(MultipathContainer.class)
2953                 .child(Multipath.class, new MultipathKey(adfBuilder.getAfi(), adfBuilder.getSafi())).create(Bgp.class);
2954         InstanceIdentifierBuilder<Vrfs> iib3 =
2955                 iid6.child(VrfsContainer.class).child(Vrfs.class, new VrfsKey(rd)).builder();
2956         InstanceIdentifier<Vrfs> iidFinal = iib3.build();
2957
2958         //** update or delete the vrfs with the rest of AddressFamilies already present in the last list
2959         AddressFamiliesVrf adfToDel = adfBuilder.build();
2960         List<AddressFamiliesVrf> adfListOriginal = new ArrayList<>(vrfOriginal.nonnullAddressFamiliesVrf());
2961         List<AddressFamiliesVrf> adfListToRemoveFromOriginal = new ArrayList<>();
2962         adfListOriginal.forEach(adf -> {
2963             if (adf.equals(adfToDel)) {
2964                 adfListToRemoveFromOriginal.add(adfToDel);
2965                 return;
2966             }
2967         });
2968         for (AddressFamiliesVrf adfToRemove : adfListToRemoveFromOriginal) {
2969             adfListOriginal.remove(adfToRemove);
2970             try {
2971                 SingleTransactionDataBroker.syncWrite(dataBroker,
2972                         LogicalDatastoreType.CONFIGURATION, iid, vrfOriginal);
2973             } catch (TransactionCommitFailedException e) {
2974                 LOG.error("delVrf: Error updating VRF to datastore", e);
2975                 throw new RuntimeException(e);
2976             }
2977         }
2978         if (adfListOriginal.isEmpty()) {
2979             LOG.debug("delVrf: delete iid: {}", iidFinal);
2980             delete(iidFinal);
2981             return true;
2982         }
2983         // not all is removed
2984         return false;
2985     }
2986
2987     public void setMultipathStatus(af_afi afi, af_safi safi, boolean enable) {
2988         long lafi = afi.getValue();
2989         long lsafi = safi.getValue();
2990
2991         InstanceIdentifier.InstanceIdentifierBuilder<Multipath> iib =
2992                 InstanceIdentifier
2993                         .builder(Bgp.class).child(MultipathContainer.class)
2994                         .child(Multipath.class,
2995                                 new MultipathKey(Long.valueOf(afi.getValue()), Long.valueOf(safi.getValue())));
2996
2997         Multipath dto = new MultipathBuilder().setAfi(lafi).setSafi(lsafi).setMultipathEnabled(enable).build();
2998         update(iib.build(), dto);
2999     }
3000
3001     public void setMultipaths(String rd, int maxpath) {
3002         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
3003                 InstanceIdentifier
3004                         .builder(Bgp.class).child(VrfMaxpathContainer.class)
3005                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
3006
3007         VrfMaxpath dto = new VrfMaxpathBuilder().setRd(rd).setMaxpaths(maxpath).build();
3008         update(iib.build(), dto);
3009     }
3010
3011     public void delMultipaths(String rd) {
3012         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
3013                 InstanceIdentifier.builder(Bgp.class).child(VrfMaxpathContainer.class)
3014                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
3015         InstanceIdentifier<VrfMaxpath> iid = iib.build();
3016         delete(iid);
3017     }
3018
3019     /*
3020     * Remove Stale Marked Routes after timer expiry.
3021     */
3022     private class RouteCleanup implements Callable<Integer> {
3023
3024         @Override
3025         public Integer call() {
3026             totalCleared = 0;
3027             try {
3028                 if (staledFibEntriesMap.isEmpty()) {
3029                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
3030                 } else {
3031                     for (String rd : staledFibEntriesMap.keySet()) {
3032                         if (Thread.interrupted()) {
3033                             return 0;
3034                         }
3035                         Map<String, Uint32> map = staledFibEntriesMap.get(rd);
3036                         if (map != null) {
3037                             for (String key : map.keySet()) {
3038                                 if (Thread.interrupted()) {
3039                                     return 0;
3040                                 }
3041                                 String prefix = extractPrefix(key);
3042                                 String nextHop = extractNextHop(key);
3043                                 totalCleared++;
3044                                 LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}, nextHop:{}",
3045                                         rd, prefix, nextHop);
3046                                 fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix, nextHop);
3047                             }
3048                         }
3049                     }
3050                 }
3051             } finally {
3052                 staledFibEntriesMap.clear();
3053             }
3054             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
3055             return 0;
3056         }
3057     }
3058
3059     /*
3060      * BGP restart scenario, ODL-BGP manager was/is running.
3061      * On re-sync notification, Get a copy of FIB database.
3062      */
3063     public void createStaleFibMap() {
3064         totalStaledCount = 0;
3065         try {
3066             staledFibEntriesMap.clear();
3067             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
3068
3069             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
3070                     LogicalDatastoreType.CONFIGURATION, id);
3071             if (fibEntries.isPresent()) {
3072                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
3073                 for (VrfTables vrfTable : staleVrfTables) {
3074                     Map<String, Uint32> staleFibEntMap = new HashMap<>();
3075                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
3076                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
3077                             //Stale marking and cleanup is only meant for the routes learned through BGP.
3078                             continue;
3079                         }
3080                         if (Thread.interrupted()) {
3081                             break;
3082                         }
3083                         totalStaledCount++;
3084                         //Create MAP from staleVrfTables.
3085                         vrfEntry.getRoutePaths()
3086                                 .forEach(
3087                                     routePath -> staleFibEntMap.put(
3088                                             appendNextHopToPrefix(vrfEntry.getDestPrefix(),
3089                                                     routePath.getNexthopAddress()), routePath.getLabel()));
3090                     }
3091                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), staleFibEntMap);
3092                 }
3093             } else {
3094                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
3095             }
3096         } catch (ReadFailedException e) {
3097             LOG.error("createStaleFibMap:: error ", e);
3098         }
3099         LOG.error("created {} staled entries ", totalStaledCount);
3100     }
3101
3102     /*
3103      * BGP config remove scenario, Need to remove all the
3104      * external routes from FIB.
3105      */
3106     public void deleteExternalFibRoutes() {
3107         totalExternalRoutes = 0;
3108         totalExternalMacRoutes = 0;
3109         try {
3110             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
3111
3112             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
3113                     LogicalDatastoreType.CONFIGURATION, id);
3114             if (fibEntries.isPresent()) {
3115                 if (fibEntries.get().getVrfTables() == null) {
3116                     LOG.error("deleteExternalFibRoutes::getVrfTables is null");
3117                     return;
3118                 }
3119                 List<VrfTables> staleVrfTables = fibEntries.get().getVrfTables();
3120                 for (VrfTables vrfTable : staleVrfTables) {
3121                     String rd = vrfTable.getRouteDistinguisher();
3122                     if (vrfTable.getVrfEntry() != null) {
3123                         for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
3124                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
3125                                 //route cleanup is only meant for the routes learned through BGP.
3126                                 continue;
3127                             }
3128                             totalExternalRoutes++;
3129                             fibDSWriter.removeFibEntryFromDS(rd, vrfEntry.getDestPrefix());
3130                         }
3131                     } else if (vrfTable.getMacVrfEntry() != null) {
3132                         for (MacVrfEntry macEntry : vrfTable.getMacVrfEntry()) {
3133                             if (RouteOrigin.value(macEntry.getOrigin()) != RouteOrigin.BGP) {
3134                                 //route cleanup is only meant for the routes learned through BGP.
3135                                 continue;
3136                             }
3137                             totalExternalMacRoutes++;
3138                             fibDSWriter.removeMacEntryFromDS(rd, macEntry.getMac());
3139                         }
3140                     }
3141                 }
3142             } else {
3143                 LOG.error("deleteExternalFibRoutes:: FIBentries.class is not present");
3144             }
3145         } catch (ReadFailedException e) {
3146             LOG.error("deleteExternalFibRoutes:: error ", e);
3147         }
3148         LOG.debug("deleted {} fib entries {} mac entries", totalExternalRoutes, totalExternalMacRoutes);
3149     }
3150
3151     public boolean addToRt2TepMap(String rd, String tepIp, String mac, Uint32 l2vni) {
3152         boolean isFirstMacUpdateFromTep = false;
3153         if (rt2TepMap.containsKey(rd)) {
3154             if (rt2TepMap.get(rd).containsKey(tepIp)) {
3155                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and tep-ip {}. No Elan DS write required",
3156                         mac, l2vni, rd, tepIp);
3157                 rt2TepMap.get(rd).get(tepIp).put(mac, l2vni);
3158             } else {
3159                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and new tep-ip {}",
3160                         mac, l2vni, rd, tepIp);
3161                 isFirstMacUpdateFromTep = true;
3162                 Map<String, Uint32> macList = new HashMap<>();
3163                 macList.put(mac, l2vni);
3164                 rt2TepMap.get(rd).put(tepIp, macList);
3165             }
3166         } else {
3167             LOG.debug("RT2 with mac {} l2vni {} from new rd {} and tep ip {}",
3168                     mac, l2vni, rd, tepIp);
3169             isFirstMacUpdateFromTep = true;
3170             Map<String, Uint32> macList = new HashMap<>();
3171             macList.put(mac, l2vni);
3172             Map<String, Map<String, Uint32>> tepIpMacMap = new HashMap<>();
3173             tepIpMacMap.put(tepIp, macList);
3174             rt2TepMap.put(rd, tepIpMacMap);
3175         }
3176         return isFirstMacUpdateFromTep;
3177     }
3178
3179     public boolean deleteFromRt2TepMap(String rd, String tepIp, String mac) {
3180         boolean isLastMacUpdateFromTep = false;
3181         LOG.debug("RT2 withdraw with rd {} mac {} tep-ip {} ", rd, mac, tepIp);
3182         if (rt2TepMap.containsKey(rd)) {
3183             if (rt2TepMap.get(rd).containsKey(tepIp)) {
3184                 if (rt2TepMap.get(rd).get(tepIp).containsKey(mac)) {
3185                     LOG.debug("RT2 Withdraw : Removing the mac {} from Map", mac);
3186                     rt2TepMap.get(rd).get(tepIp).remove(mac);
3187                     if (rt2TepMap.get(rd).get(tepIp).isEmpty()) {
3188                         isLastMacUpdateFromTep = true;
3189                         LOG.debug("RT2 Withdraw : Removing the tep-ip {} from Map", tepIp);
3190                         rt2TepMap.get(rd).remove(tepIp);
3191                         if (rt2TepMap.get(rd).isEmpty()) {
3192                             LOG.debug("RT2 Withdraw : Removing the rd {} from Map", rd);
3193                             rt2TepMap.remove(rd);
3194                         }
3195                     }
3196                 }
3197             }
3198         }
3199         return isLastMacUpdateFromTep;
3200     }
3201
3202     public Collection<String> getTepIPs(String rd) {
3203         final Map<String, Map<String, Uint32>> tepIpMap = rt2TepMap.get(rd);
3204         return tepIpMap != null ? tepIpMap.keySet() : Collections.emptyList();
3205     }
3206
3207     public boolean isBgpConnected() {
3208         return (bgpRouter == null) ? false : bgpRouter.isBgpConnected();
3209     }
3210
3211     public long getLastConnectedTS() {
3212         return (bgpRouter == null) ? 0 : bgpRouter.getLastConnectedTS();
3213     }
3214
3215     public long getConnectTS() {
3216         return (bgpRouter == null) ? 0 : bgpRouter.getConnectTS();
3217     }
3218
3219     public long getStartTS() {
3220         return (bgpRouter == null) ? 0 : bgpRouter.getStartTS();
3221     }
3222
3223     public TTransport getTransport() {
3224         return bgpRouter.getTransport();
3225     }
3226
3227     public int getTotalStaledCount() {
3228         return totalStaledCount;
3229     }
3230
3231     public int getTotalCleared() {
3232         return totalCleared;
3233     }
3234
3235     public static List<Neighbors> getNbrList() {
3236         return nbrList;
3237     }
3238
3239     public BgpCounters getBgpCounters() {
3240         return bgpCountersReference.get();
3241     }
3242
3243     private void startBgpCountersTask() {
3244         if (getBgpCounters() == null && bgpCountersReference.compareAndSet(null,
3245                 new BgpCounters(getBgpSdncMipIp(), metricProvider))) {
3246             bgpCountersTask = executor.scheduleAtFixedRate(bgpCountersReference.get(), 0, 120 * 1000,
3247                     TimeUnit.MILLISECONDS);
3248             LOG.info("Bgp Counters task scheduled for every two minutes.");
3249
3250             bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
3251         }
3252     }
3253
3254     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3255             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3256     private void stopBgpCountersTask() {
3257         final BgpCounters bgpCounters = bgpCountersReference.getAndSet(null);
3258         if (bgpCounters != null) {
3259             bgpCountersTask.cancel(true);
3260             bgpCounters.close();
3261         }
3262     }
3263
3264     private void startBgpAlarmsTask() {
3265         if (getBgpAlarms() == null && bgpAlarmsReference.compareAndSet(null, new BgpAlarms(this))) {
3266             bgpAlarmsReference.get().init();
3267             bgpAlarmsTask = executor.scheduleAtFixedRate(bgpAlarmsReference.get(), 0, 60 * 1000, TimeUnit.MILLISECONDS);
3268             LOG.info("Bgp Alarms task scheduled for every minute.");
3269         } else {
3270             LOG.trace("Bgp Alarms task already scheduled for every minute.");
3271         }
3272     }
3273
3274     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3275             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3276     private void stopBgpAlarmsTask() {
3277         final BgpAlarms bgpAlarms = bgpAlarmsReference.getAndSet(null);
3278         if (bgpAlarms != null) {
3279             bgpAlarmsTask.cancel(true);
3280             bgpAlarms.close();
3281         }
3282     }
3283
3284     public BgpAlarms getBgpAlarms() {
3285         return bgpAlarmsReference.get();
3286     }
3287
3288     public void getPeerStatus(String nbrIp, long nbrAsNum) throws
3289             BgpRouterException, TException {
3290         bgpRouter.getPeerStatus(nbrIp, nbrAsNum);
3291     }
3292
3293     private static String appendNextHopToPrefix(String prefix, String nextHop) {
3294         return prefix + ":" + nextHop;
3295     }
3296
3297     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3298             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3299     private static String extractPrefix(String prefixNextHop) {
3300         return prefixNextHop.split(":")[0];
3301     }
3302
3303     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3304             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3305     private static String extractNextHop(String prefixNextHop) {
3306         return prefixNextHop.split(":")[1];
3307     }
3308
3309     private static String extractMd5Secret(final Neighbors val) {
3310         String md5Secret = null;
3311         TcpSecurityOption tcpSecOpt = val.getTcpSecurityOption();
3312         if (tcpSecOpt != null) {
3313             if (tcpSecOpt instanceof TcpMd5SignatureOption) {
3314                 md5Secret = ((TcpMd5SignatureOption) tcpSecOpt).getTcpMd5SignaturePassword().getValue();
3315             } else { // unknown TcpSecurityOption
3316                 LOG.debug("neighbors  Ignored unknown tcp-security-option of peer {}", val.getAddress().getValue());
3317             }
3318         }
3319         return md5Secret;
3320     } // private method extractMd5Secret
3321
3322     @SuppressWarnings("checkstyle:IllegalCatch")
3323     @Override
3324     public ListenableFuture<RpcResult<InitiateEorOutput>> initiateEor(InitiateEorInput input) {
3325         boolean returnError = false;
3326         String msg = null;
3327         String neighborIp = null;
3328         if (!isBGPEntityOwner()) {
3329             msg = String.format("RPC triggered in Non-EoS Owner");
3330             return Futures.immediateFuture(
3331                     RpcResultBuilder.<InitiateEorOutput>failed().withError(RpcError.ErrorType.APPLICATION,
3332                             msg).build());
3333         }
3334         if (input == null) {
3335             msg = String.format("BGP invalid input for EoR");
3336             LOG.error("Error : {}", msg);
3337             returnError = true;
3338         } else {
3339             neighborIp = input.getNeighborIp();
3340         }
3341         if (eorSupressedDuetoUpgradeFlag.get() == false) {
3342             msg = String.format("EoR triggerd by RBU-RPC call before replay"
3343                     + "of BGP configuration (or) BGP not restarted");
3344             LOG.error("Error : {}", msg);
3345         }
3346         if ("ALL".compareToIgnoreCase(neighborIp) == 0) {
3347             //send EoR for all the neighbor
3348             LOG.error("EoR trigger received to ALL neighbors");
3349             final int numberOfEORRetries = 3;
3350             RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
3351             do {
3352                 try {
3353                     BgpRouter br = bgpRouter;
3354                     br.sendEOR();
3355                     LOG.debug("RPC: sendEOR {} successful", br);
3356                     break;
3357                 } catch (Exception e) {
3358                     eorRetry.errorOccured();
3359                     LOG.error("Replay:sedEOR() received exception:", e);
3360                 }
3361             } while (eorRetry.shouldRetry());
3362             eorSupressedDuetoUpgradeFlag.set(false);
3363         } else if (InetAddresses.isInetAddress(neighborIp)) {
3364             //send EoR for only one neighbor
3365             msg = String.format("Inidividual neighbors EoR is not supported");
3366             LOG.warn("Error : {}", msg);
3367             returnError = true;
3368         } else {
3369             //error
3370             msg = String.format("RPC: initiateEor: Invalid input ");
3371             LOG.warn("Error : {}", msg);
3372             returnError = true;
3373         }
3374         if (returnError) {
3375             return Futures.immediateFuture(
3376                     RpcResultBuilder.<InitiateEorOutput>failed().withError(RpcError.ErrorType.APPLICATION,
3377                             msg).build());
3378         }
3379         InitiateEorOutput initiateEorOutput =
3380                 new InitiateEorOutputBuilder().setRetVal(0L).build();
3381         return Futures.immediateFuture(RpcResultBuilder.<InitiateEorOutput>success()
3382                 .withResult(initiateEorOutput).build());
3383     }
3384 }