bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.bgpmanager;
   9
  10 import static org.opendaylight.netvirt.bgpmanager.oam.BgpConstants.HISTORY_LIMIT;
  11 import static org.opendaylight.netvirt.bgpmanager.oam.BgpConstants.HISTORY_THRESHOLD;
  12
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.net.InetAddresses;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.ThreadFactoryBuilder;
  18 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  19 import io.netty.util.concurrent.GlobalEventExecutor;
  20 import java.lang.reflect.Constructor;
  21 import java.lang.reflect.InvocationTargetException;
  22 import java.net.InetAddress;
  23 import java.net.InetSocketAddress;
  24 import java.net.NetworkInterface;
  25 import java.net.SocketException;
  26 import java.util.ArrayList;
  27 import java.util.Collection;
  28 import java.util.Collections;
  29 import java.util.Enumeration;
  30 import java.util.HashMap;
  31 import java.util.Iterator;
  32 import java.util.List;
  33 import java.util.Map;
  34 import java.util.Objects;
  35 import java.util.Optional;
  36 import java.util.Timer;
  37 import java.util.concurrent.Callable;
  38 import java.util.concurrent.ConcurrentHashMap;
  39 import java.util.concurrent.CountDownLatch;
  40 import java.util.concurrent.ExecutionException;
  41 import java.util.concurrent.Executors;
  42 import java.util.concurrent.Future;
  43 import java.util.concurrent.ScheduledExecutorService;
  44 import java.util.concurrent.ScheduledFuture;
  45 import java.util.concurrent.TimeUnit;
  46 import java.util.concurrent.TimeoutException;
  47 import java.util.concurrent.atomic.AtomicBoolean;
  48 import java.util.concurrent.atomic.AtomicInteger;
  49 import java.util.concurrent.atomic.AtomicReference;
  50 import javax.annotation.PreDestroy;
  51 import javax.inject.Inject;
  52 import javax.inject.Singleton;
  53 import org.apache.thrift.TApplicationException;
  54 import org.apache.thrift.TException;
  55 import org.apache.thrift.transport.TTransport;
  56 import org.eclipse.jdt.annotation.Nullable;
  57 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  60 import org.opendaylight.infrautils.metrics.MetricProvider;
  61 import org.opendaylight.mdsal.binding.api.ClusteredDataTreeChangeListener;
  62 import org.opendaylight.mdsal.binding.api.DataBroker;
  63 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  64 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
  65 import org.opendaylight.mdsal.eos.binding.api.Entity;
  66 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipCandidateRegistration;
  67 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipListenerRegistration;
  68 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  69 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  70 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  71 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  72 import org.opendaylight.netvirt.bgpmanager.commands.ClearBgpCli;
  73 import org.opendaylight.netvirt.bgpmanager.oam.BgpAlarms;
  74 import org.opendaylight.netvirt.bgpmanager.oam.BgpConstants;
  75 import org.opendaylight.netvirt.bgpmanager.oam.BgpCounters;
  76 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouter;
  77 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpRouterException;
  78 import org.opendaylight.netvirt.bgpmanager.thrift.client.BgpSyncHandle;
  79 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Routes;
  80 import org.opendaylight.netvirt.bgpmanager.thrift.gen.Update;
  81 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_afi;
  82 import org.opendaylight.netvirt.bgpmanager.thrift.gen.af_safi;
  83 import org.opendaylight.netvirt.bgpmanager.thrift.gen.protocol_type;
  84 import org.opendaylight.netvirt.bgpmanager.thrift.gen.qbgpConstants;
  85 import org.opendaylight.netvirt.bgpmanager.thrift.server.BgpThriftService;
  86 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  87 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  88 import org.opendaylight.ovsdb.utils.mdsal.utils.TransactionHistory;
  89 import org.opendaylight.ovsdb.utils.mdsal.utils.TransactionType;
  90 import org.opendaylight.serviceutils.tools.listener.AbstractAsyncDataTreeChangeListener;
  91 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebfd.rev190219.BfdConfig;
  92 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebfd.rev190219.BfdConfigBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.AddressFamily;
  94 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.Bgp;
  95 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.BgpControlPlaneType;
  96 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EbgpService;
  97 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.EncapType;
  98 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorInput;
  99 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorOutput;
 100 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.InitiateEorOutputBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.TcpMd5SignaturePasswordType;
 102 //import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.*;
 103 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsId;
 104 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.AsIdBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServer;
 106 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.ConfigServerBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.DcgwTepList;
 108 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestart;
 109 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.GracefulRestartBuilder;
 110 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.Logging;
 111 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.LoggingBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.MultipathContainer;
 113 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NeighborsContainer;
 114 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.NetworksContainer;
 115 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfMaxpathContainer;
 116 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.VrfsContainer;
 117 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTep;
 118 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTepBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.dcgw.tep.list.DcgwTepKey;
 120 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.Multipath;
 121 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.MultipathBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.multipathcontainer.MultipathKey;
 123 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.Neighbors;
 124 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.NeighborsBuilder;
 125 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.NeighborsKey;
 126 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamilies;
 127 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamiliesBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.AddressFamiliesKey;
 129 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.EbgpMultihop;
 130 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.EbgpMultihopBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.UpdateSource;
 132 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.neighborscontainer.neighbors.UpdateSourceBuilder;
 133 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.Networks;
 134 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.NetworksBuilder;
 135 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.networkscontainer.NetworksKey;
 136 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpath;
 137 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpathBuilder;
 138 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfmaxpathcontainer.VrfMaxpathKey;
 139 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.Vrfs;
 140 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.VrfsBuilder;
 141 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.VrfsKey;
 142 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.vrfs.AddressFamiliesVrf;
 143 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.vrfs.AddressFamiliesVrfBuilder;
 144 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.bgp.vrfscontainer.vrfs.AddressFamiliesVrfKey;
 145 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.TcpSecurityOption;
 146 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOption;
 147 import org.opendaylight.yang.gen.v1.urn.ericsson.params.xml.ns.yang.ebgp.rev150901.tcp.security.option.grouping.tcp.security.option.TcpMd5SignatureOptionBuilder;
 148 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
 149 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
 150 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
 151 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 152 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 153 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 154 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.macvrfentries.MacVrfEntry;
 155 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 156 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 157 import org.opendaylight.yangtools.yang.binding.DataObject;
 158 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 159 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 160 import org.opendaylight.yangtools.yang.common.RpcError;
 161 import org.opendaylight.yangtools.yang.common.RpcResult;
 162 import org.opendaylight.yangtools.yang.common.RpcResultBuilder;
 163 import org.opendaylight.yangtools.yang.common.Uint32;
 164 import org.osgi.framework.BundleContext;
 165 import org.osgi.util.tracker.ServiceTracker;
 166 import org.slf4j.Logger;
 167 import org.slf4j.LoggerFactory;
 168
 169 @Singleton
 170 public class BgpConfigurationManager implements EbgpService {
 171     private static final Logger LOG = LoggerFactory.getLogger(BgpConfigurationManager.class);
 172
 173     // to have stale FIB map (RD, Prefix)
 174     //  number of seconds wait for route sync-up between ODL and BGP
 175     private static final int BGP_RESTART_ROUTE_SYNC_SEC = 600;
 176     private static final String UPDATE_PORT = "bgp.thrift.service.port";
 177     private static final String CONFIG_HOST = "vpnservice.bgpspeaker.host.name";
 178     private static final String CONFIG_PORT = "vpnservice.bgpspeaker.thrift.port";
 179     private static final String DEF_UPORT = "6644";
 180     private static final String DEF_CHOST = "255.255.255.255"; // Invalid Host IP
 181     private static final String DEF_CPORT = "7644";
 182     private static final String DEF_BGP_SDNC_MIP = "127.0.0.1";
 183     //vpnservice.bgp.thrift.bgp.mip is the MIP present with ODL. Here we open 6644 port
 184     private static final String BGP_SDNC_MIP = "vpnservice.bgp.thrift.bgp.mip";
 185     private static final String BGP_GR_RESTART_TIMER_PROPERTY = "vpnservice.bgp.gr.timer";
 186     private static final String BGP_KA_TIMER_PROPERTY = "vpnservice.bgp.ka.timer";
 187     private static final String BGP_HOLD_TIMER_PROPERTY = "vpnservice.bgp.hold.timer";
 188     private static final String BGP_EOR_DELAY_PROPERTY = "vpnservice.bgp.eordelay";
 189     private static final int DEF_BGP_KA_TIME = 60;
 190     private static final int DEF_BGP_HOLD_TIME = 180;
 191     private static final int DEF_BGP_GR_TIME = 4000;
 192     private static final int RESTART_DEFAULT_GR = 90;
 193     private static final int DS_RETRY_COUNT = 100; //100 retries, each after WAIT_TIME_BETWEEN_EACH_TRY_MILLIS seconds
 194     private static final long WAIT_TIME_BETWEEN_EACH_TRY_MILLIS = 1000L; //one second sleep after every retry
 195     private static final String BGP_ENTITY_TYPE_FOR_OWNERSHIP = "bgp";
 196     private static final String BGP_EOR_DELAY = "vpnservice.bgp.eordelay";
 197     private static final String DEF_BGP_EOR_DELAY = "1800";
 198     private static final String BGP_ENTITY_NAME = "bgp";
 199     private static final String ADD_WARN = "Config store updated; undo with Delete if needed.";
 200     private static final String DEL_WARN = "Config store updated; undo with Add if needed.";
 201     private static final String UPD_WARN = "Update operation not supported; Config store updated;"
 202             + " restore with another Update if needed.";
 203     private static long bgp_as_num = 0;
 204     private static List<Neighbors> nbrList = new ArrayList<>();
 205     private int bgpKaTime = 0;
 206     private int bgpHoldTime = 0;
 207     private int bgpGrRestartTime = 0;
 208
 209     private static final Class<?>[] REACTORS = {
 210         ConfigServerReactor.class, AsIdReactor.class,
 211         GracefulRestartReactor.class, LoggingReactor.class,
 212         NeighborsReactor.class, UpdateSourceReactor.class,
 213         EbgpMultihopReactor.class, AddressFamiliesReactor.class,
 214         NetworksReactor.class, VrfsReactor.class, BgpReactor.class,
 215         MultipathReactor.class, VrfMaxpathReactor.class, BfdConfigReactor.class
 216     };
 217
 218     private IBgpManager bgpManager;
 219     private final DataBroker dataBroker;
 220     private final FibDSWriter fibDSWriter;
 221     private final IVpnLinkService vpnLinkService;
 222     private final BundleContext bundleContext;
 223     private final BgpUtil bgpUtil;
 224     private volatile Bgp config;
 225     private final BgpRouter bgpRouter;
 226     private final BgpSyncHandle bgpSyncHandle = new BgpSyncHandle();
 227     private volatile BgpThriftService bgpThriftService = null;
 228     private final int delayEorSeconds;
 229
 230     private final CountDownLatch initer = new CountDownLatch(1);
 231
 232     private final String hostStartup;
 233     private final String portStartup;
 234
 235     private final AtomicReference<BgpCounters> bgpCountersReference = new AtomicReference<>();
 236     private ScheduledFuture<?> bgpCountersTask;
 237
 238     private final AtomicReference<BgpAlarms> bgpAlarmsReference = new AtomicReference<>();
 239     private ScheduledFuture<?> bgpAlarmsTask;
 240
 241     private Future<?> lastReplayJobFt;
 242     private ScheduledFuture<?> routeCleanupFuture;
 243
 244     private long staleStartTime;
 245     private long staleEndTime;
 246     private long cfgReplayStartTime;
 247     private long cfgReplayEndTime;
 248     private long staleCleanupTime;
 249     private int totalStaledCount;
 250     private int totalCleared;
 251     private int totalExternalRoutes;
 252     private int totalExternalMacRoutes;
 253
 254     private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(
 255             new ThreadFactoryBuilder().setNameFormat("bgp-config-%d").setDaemon(true).build());
 256
 257     /**
 258      * this map store the new address families to send to quagga. When it is sended you must clear it.
 259      * The keys String are rd (route distinguisher).
 260      * */
 261     private final ConcurrentHashMap<String, List<AddressFamiliesVrf>> mapNewAdFamily = new ConcurrentHashMap<>();
 262
 263     // map<rd, map<prefix/len:nexthop, label>>
 264     private final Map<String, Map<String, Uint32>> staledFibEntriesMap = new ConcurrentHashMap<>();
 265
 266     // map<rd, map<tep-ip, map<mac, l2vni>>>
 267     private final Map<String, Map<String, Map<String, Uint32>>> rt2TepMap = new ConcurrentHashMap<>();
 268
 269     //map<rd+prefix/plen, list (nexthop)>
 270     private final Map<String,List> fibMap = new HashMap<>();
 271
 272     private final List<AutoCloseable> listeners = new ArrayList<>();
 273
 274     private final EntityOwnershipUtils entityOwnershipUtils;
 275     private final EntityOwnershipCandidateRegistration candidateRegistration;
 276     private final EntityOwnershipListenerRegistration entityListenerRegistration;
 277     private final MetricProvider metricProvider;
 278     private final TransactionHistory bgpUpdatesHistory;
 279
 280     private volatile AtomicBoolean eorSupressedDuetoUpgradeFlag = new AtomicBoolean(false);
 281
 282     @Inject
 283     public BgpConfigurationManager(final DataBroker dataBroker,
 284             final EntityOwnershipService entityOwnershipService,
 285             final FibDSWriter fibDSWriter,
 286             final IVpnLinkService vpnLinkSrvce,
 287             final BundleContext bundleContext,
 288             final BgpUtil bgpUtil,
 289             final MetricProvider metricProvider) {
 290         this.dataBroker = dataBroker;
 291         this.fibDSWriter = fibDSWriter;
 292         this.vpnLinkService = vpnLinkSrvce;
 293         this.bundleContext = bundleContext;
 294         this.bgpUtil = bgpUtil;
 295         this.metricProvider = metricProvider;
 296         hostStartup = getProperty(CONFIG_HOST, DEF_CHOST);
 297         portStartup = getProperty(CONFIG_PORT, DEF_CPORT);
 298         bgpKaTime =
 299                 Integer.parseInt(getProperty(BGP_KA_TIMER_PROPERTY,
 300                         Integer.toString(DEF_BGP_KA_TIME)));
 301         bgpHoldTime =
 302                 Integer.parseInt(getProperty(BGP_HOLD_TIMER_PROPERTY,
 303                         Integer.toString(DEF_BGP_HOLD_TIME)));
 304         bgpGrRestartTime =
 305                 Integer.parseInt(getProperty(BGP_GR_RESTART_TIMER_PROPERTY,
 306                         Integer.toString(DEF_BGP_GR_TIME)));
 307         LOG.info("ConfigServer at {}:{}", hostStartup, portStartup);
 308         VtyshCli.setHostAddr(hostStartup);
 309         ClearBgpCli.setHostAddr(hostStartup);
 310         bgpUpdatesHistory = new TransactionHistory(HISTORY_LIMIT, HISTORY_THRESHOLD);
 311         bgpRouter = BgpRouter.newInstance(this::getConfig, this::isBGPEntityOwner, bgpUpdatesHistory);
 312         delayEorSeconds = Integer.parseInt(getProperty(BGP_EOR_DELAY, DEF_BGP_EOR_DELAY));
 313
 314         entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 315
 316         candidateRegistration = registerEntityCandidate(entityOwnershipService);
 317         entityListenerRegistration = registerEntityListener(entityOwnershipService);
 318
 319         LOG.info("BGP Configuration manager initialized");
 320         initer.countDown();
 321
 322         GlobalEventExecutor.INSTANCE.execute(() -> {
 323             ServiceTracker<IBgpManager, ?> tracker = null;
 324             try {
 325                 tracker = new ServiceTracker<>(bundleContext, IBgpManager.class, null);
 326                 tracker.open();
 327                 bgpManager = (IBgpManager) tracker.waitForService(TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES));
 328                 Preconditions.checkState(bgpManager != null, "IBgpManager service not found");
 329             } catch (InterruptedException e) {
 330                 throw new IllegalStateException("Error retrieving IBgpManager service", e);
 331             } finally {
 332                 if (tracker != null) {
 333                     tracker.close();
 334                 }
 335             }
 336
 337             String updatePort = getProperty(UPDATE_PORT, DEF_UPORT);
 338             if (InetAddresses.isInetAddress(getBgpSdncMipIp())) {
 339                 InetSocketAddress bgpThriftServerSocketAddr = new InetSocketAddress(getBgpSdncMipIp(),
 340                         Integer.parseInt(updatePort));
 341                 bgpThriftService = new BgpThriftService(bgpThriftServerSocketAddr, bgpManager, this);
 342                 if (isBGPEntityOwner()) {
 343                     //I am EoS owner of BGP, opening bgp thrift UPDATE-SERVER port.
 344                     LOG.info("BGP Configuration manager initialized: UPDATE-SERVER started");
 345                     bgpThriftService.start();
 346                 }
 347                 LOG.info("UPDATE server started :ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 348             } else {
 349                 LOG.error("Failed to init UPDATE server invalid ip:port={}:{}", getBgpSdncMipIp(), updatePort);
 350             }
 351         });
 352         registerCallbacks();
 353         LOG.info("BgpConfigurationManager initialized. IBgpManager={}", bgpManager);
 354     }
 355
 356     public String getBgpSdncMipIp() {
 357         return getProperty(BGP_SDNC_MIP, DEF_BGP_SDNC_MIP);
 358     }
 359
 360     public long getStaleCleanupTime() {
 361         return staleCleanupTime;
 362     }
 363
 364     public void setStaleCleanupTime(long staleCleanupTime) {
 365         this.staleCleanupTime = staleCleanupTime;
 366     }
 367
 368     public long getCfgReplayEndTime() {
 369         return cfgReplayEndTime;
 370     }
 371
 372     public void setCfgReplayEndTime(long cfgReplayEndTime) {
 373         this.cfgReplayEndTime = cfgReplayEndTime;
 374     }
 375
 376     public TransactionHistory getBgpUpdatesHistory() {
 377         return bgpUpdatesHistory;
 378     }
 379
 380     public long getCfgReplayStartTime() {
 381         return cfgReplayStartTime;
 382     }
 383
 384     public void setCfgReplayStartTime(long cfgReplayStartTime) {
 385         this.cfgReplayStartTime = cfgReplayStartTime;
 386     }
 387
 388     public long getStaleEndTime() {
 389         return staleEndTime;
 390     }
 391
 392     public void setStaleEndTime(long staleEndTime) {
 393         this.staleEndTime = staleEndTime;
 394     }
 395
 396     public long getStaleStartTime() {
 397         return staleStartTime;
 398     }
 399
 400     public void setStaleStartTime(long staleStartTime) {
 401         this.staleStartTime = staleStartTime;
 402     }
 403
 404     private Object createListener(Class<?> cls) {
 405         try {
 406             Constructor<?> ctor = cls.getConstructor(BgpConfigurationManager.class);
 407             return ctor.newInstance(this);
 408         } catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException
 409                 e) {
 410             LOG.error("Failed to create listener object", e);
 411             return null;
 412         }
 413     }
 414
 415     private void registerCallbacks() {
 416         for (Class<?> reactor : REACTORS) {
 417             Object obj = createListener(reactor);
 418             if (obj != null) {
 419                 AbstractAsyncDataTreeChangeListener dcl = (AbstractAsyncDataTreeChangeListener) obj;
 420                 dcl.register();
 421                 listeners.add(dcl);
 422             }
 423         }
 424     }
 425
 426     @SuppressWarnings("checkstyle:IllegalCatch")
 427     @PreDestroy
 428     public void close() {
 429         executor.shutdown();
 430
 431         if (bgpThriftService != null) {
 432             bgpThriftService.stop();
 433             bgpThriftService = null;
 434         }
 435
 436         if (isBgpConnected()) {
 437             //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 438             bgpRouter.disconnect();
 439         }
 440
 441         if (candidateRegistration != null) {
 442             candidateRegistration.close();
 443         }
 444
 445         entityListenerRegistration.close();
 446
 447         listeners.forEach(l -> {
 448             try {
 449                 l.close();
 450             } catch (Exception e) {
 451                 LOG.warn("Error closing {}", l ,e);
 452             }
 453         });
 454
 455         LOG.info("{} close", getClass().getSimpleName());
 456     }
 457
 458     private String getProperty(String var, String def) {
 459         String property = bundleContext.getProperty(var);
 460         return property == null ? def : property;
 461     }
 462
 463     private EntityOwnershipCandidateRegistration registerEntityCandidate(
 464             final EntityOwnershipService entityOwnershipService) {
 465         try {
 466             return entityOwnershipService.registerCandidate(
 467                     new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME));
 468         } catch (CandidateAlreadyRegisteredException e) {
 469             LOG.error("failed to register bgp entity", e);
 470             return null;
 471         }
 472     }
 473
 474     private EntityOwnershipListenerRegistration registerEntityListener(
 475             final EntityOwnershipService entityOwnershipService) {
 476         return entityOwnershipService.registerListener(BGP_ENTITY_TYPE_FOR_OWNERSHIP, ownershipChange -> {
 477             LOG.info("entity owner change event fired: {}", ownershipChange);
 478
 479             if (ownershipChange.getState() == EntityOwnershipChangeState.LOCAL_OWNERSHIP_GRANTED) {
 480                 LOG.info("This PL is the Owner");
 481                 if (bgpThriftService != null) {
 482                     //opening UPDATE-SERVER port.
 483                     bgpThriftService.start();
 484                 } else {
 485                     LOG.error("I am the owner of BGP entity, but bgpThriftService is not initialized yet");
 486                 }
 487                 bgpRestarted();
 488             } else {
 489                 LOG.info("Not owner: hasOwner: {}, isOwner: {}", ownershipChange.getState().hasOwner(),
 490                         ownershipChange.getState().isOwner());
 491                 if (bgpThriftService != null && bgpThriftService.isBgpThriftServiceStarted()) {
 492                     //close the bgp Thrift Update-SERVER port opened on non-Entity Owner
 493                     bgpThriftService.stop();
 494                 }
 495                 if (isBgpConnected()) {
 496                     //disconnect the CONFIG SERVER port (which was )opened during I was Owner
 497                     bgpRouter.disconnect();
 498                 }
 499                 stopBgpCountersTask();
 500                 stopBgpAlarmsTask();
 501             }
 502         });
 503     }
 504
 505     public boolean isBGPEntityOwner() {
 506         if (entityOwnershipUtils == null) {
 507             LOG.error("entityOwnershipUtils is NULL when listener callbacks fired");
 508             return false;
 509         }
 510         return entityOwnershipUtils.isEntityOwner(new Entity(BGP_ENTITY_TYPE_FOR_OWNERSHIP, BGP_ENTITY_NAME), 0, 1);
 511     }
 512
 513     public Bgp get() {
 514         config = getConfig();
 515         return config;
 516     }
 517
 518     public class ConfigServerReactor
 519             extends AbstractAsyncDataTreeChangeListener<ConfigServer>
 520             implements ClusteredDataTreeChangeListener<ConfigServer> {
 521         private static final String YANG_OBJ = "config-server ";
 522
 523         public ConfigServerReactor() {
 524             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 525                     InstanceIdentifier.create(Bgp.class).child(ConfigServer.class),
 526                     org.opendaylight.infrautils.utils.concurrent.Executors
 527                             .newListeningSingleThreadExecutor("ConfigServerReactor", LOG));
 528         }
 529
 530         @Override
 531         public void add(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 532             LOG.trace("received bgp connect config host {}", val.getHost().getValue());
 533             if (!isBGPEntityOwner()) {
 534                 return;
 535             }
 536
 537             try {
 538                 initer.await();
 539             } catch (InterruptedException e) {
 540                 // Ignored
 541             }
 542             LOG.debug("issueing bgp router connect to host {}", val.getHost().getValue());
 543             bgpRouter.configServerUpdated();
 544             synchronized (BgpConfigurationManager.this) {
 545                 boolean res = bgpRouter.connect(val.getHost().getValue(),
 546                         val.getPort().intValue());
 547                 if (!res) {
 548                     LOG.error(YANG_OBJ + "Add failed; " + ADD_WARN);
 549                 }
 550             }
 551             VtyshCli.setHostAddr(val.getHost().getValue());
 552             ClearBgpCli.setHostAddr(val.getHost().getValue());
 553         }
 554
 555         @Override
 556         public void remove(InstanceIdentifier<ConfigServer> iid, ConfigServer val) {
 557             LOG.trace("received bgp disconnect");
 558             if (!isBGPEntityOwner()) {
 559                 return;
 560             }
 561
 562             bgpRouter.configServerUpdated();
 563
 564             synchronized (BgpConfigurationManager.this) {
 565                 if (bgp_as_num != 0) {
 566                     try {
 567                         bgpRouter.stopBgp(bgp_as_num);
 568                         stopBgpCountersTask();
 569                         stopBgpAlarmsTask();
 570                     } catch (TException | BgpRouterException e) {
 571                         LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 572                     }
 573                 } else {
 574                     LOG.debug("bgp as-id is null while removing config-server");
 575                 }
 576                 bgpRouter.disconnect();
 577             }
 578         }
 579
 580         @Override
 581         public void update(InstanceIdentifier<ConfigServer> iid,
 582                 ConfigServer oldval, ConfigServer newval) {
 583             LOG.trace("received bgp Connection update");
 584             if (!isBGPEntityOwner()) {
 585                 return;
 586             }
 587             LOG.error(YANG_OBJ + UPD_WARN);
 588         }
 589     }
 590
 591     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
 592             justification = "https://github.com/spotbugs/spotbugs/issues/811")
 593     private BgpRouter getClient(String yangObj) {
 594         if (bgpRouter == null || !bgpRouter.isBgpConnected()) {
 595             LOG.warn("{}: configuration received when BGP is inactive", yangObj);
 596             return null;
 597         }
 598         return bgpRouter;
 599     }
 600
 601     public class AsIdReactor extends AbstractAsyncDataTreeChangeListener<AsId>
 602             implements ClusteredDataTreeChangeListener<AsId> {
 603
 604         private static final String YANG_OBJ = "as-id ";
 605
 606         public AsIdReactor() {
 607             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 608                     InstanceIdentifier.create(Bgp.class).child(AsId.class),
 609                     org.opendaylight.infrautils.utils.concurrent.Executors
 610                             .newListeningSingleThreadExecutor("AsIdReactor", LOG));
 611         }
 612
 613         @SuppressFBWarnings(value  = "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD")
 614         @Override
 615         public void add(InstanceIdentifier<AsId> iid, AsId val) {
 616             LOG.error("received bgp add asid {}", val);
 617             if (!isBGPEntityOwner()) {
 618                 return;
 619             }
 620             LOG.debug("received add router config asNum {}", val.getLocalAs());
 621             bgp_as_num = val.getLocalAs().longValue();
 622             synchronized (BgpConfigurationManager.this) {
 623                 BgpRouter br = getClient(YANG_OBJ);
 624                 if (br == null) {
 625                     LOG.debug("{} Unable to process add for asNum {}; {} {}", YANG_OBJ, val.getLocalAs(),
 626                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 627                     return;
 628                 }
 629                 bgpRestarted();
 630
 631                 startBgpCountersTask();
 632                 startBgpAlarmsTask();
 633             }
 634         }
 635
 636         @Override
 637         public void remove(InstanceIdentifier<AsId> iid, AsId val) {
 638             LOG.error("received delete router config asNum {}", val.getLocalAs());
 639             if (!isBGPEntityOwner()) {
 640                 return;
 641             }
 642             synchronized (BgpConfigurationManager.this) {
 643                 long asNum = val.getLocalAs().toJava();
 644                 BgpRouter br = getClient(YANG_OBJ);
 645                 bgp_as_num = 0;
 646                 if (br == null) {
 647                     LOG.debug("{} Unable to process remove for asNum {}; {} {}", YANG_OBJ, asNum,
 648                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 649                     return;
 650                 }
 651                 try {
 652                     br.stopBgp(asNum);
 653                 } catch (TException | BgpRouterException e) {
 654                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 655                 }
 656
 657                 stopBgpCountersTask();
 658                 stopBgpAlarmsTask();
 659
 660                 Bgp conf = getConfig();
 661                 if (conf == null) {
 662                     LOG.error("Config Null while removing the as-id");
 663                     return;
 664                 }
 665                 LOG.debug("Removing external routes from FIB");
 666                 deleteExternalFibRoutes();
 667                 Map<NeighborsKey, Neighbors> keyNeighborsMap = conf.getNeighborsContainer() == null ? null
 668                         : conf.getNeighborsContainer().getNeighbors();
 669                 if (keyNeighborsMap != null && keyNeighborsMap.size() > 0) {
 670                     LOG.error("Tring to remove the as-id when neighbor config is already present");
 671                     for (Neighbors nbr : keyNeighborsMap.values()) {
 672                         LOG.debug("Removing Neighbor {} from Data store", nbr.getAddress().getValue());
 673                         delNeighbor(nbr.getAddress().getValue());
 674                     }
 675                 }
 676             }
 677         }
 678
 679         @Override
 680         public void update(InstanceIdentifier<AsId> iid,
 681                 AsId oldval, AsId newval) {
 682             if (!isBGPEntityOwner()) {
 683                 return;
 684             }
 685             LOG.error(YANG_OBJ + UPD_WARN);
 686         }
 687     }
 688
 689     public class GracefulRestartReactor extends AbstractAsyncDataTreeChangeListener<GracefulRestart>
 690             implements ClusteredDataTreeChangeListener<GracefulRestart> {
 691
 692         private static final String YANG_OBJ = "graceful-restart ";
 693
 694         public GracefulRestartReactor() {
 695             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 696                     InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class),
 697                     org.opendaylight.infrautils.utils.concurrent.Executors
 698                             .newListeningSingleThreadExecutor("GracefulRestartReactor", LOG));
 699         }
 700
 701         @Override
 702         public void add(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 703             if (!isBGPEntityOwner()) {
 704                 return;
 705             }
 706             synchronized (BgpConfigurationManager.this) {
 707                 int stalePathTime = val.getStalepathTime().intValue();
 708                 BgpRouter br = getClient(YANG_OBJ);
 709                 if (br == null) {
 710                     LOG.error("{} Unable to add stale-path time {}; {} {}", YANG_OBJ, stalePathTime,
 711                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 712                     return;
 713                 }
 714                 try {
 715                     br.addGracefulRestart(stalePathTime);
 716                 } catch (TException | BgpRouterException e) {
 717                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 718                 }
 719             }
 720         }
 721
 722         @Override
 723         public void remove(InstanceIdentifier<GracefulRestart> iid, GracefulRestart val) {
 724             if (!isBGPEntityOwner()) {
 725                 return;
 726             }
 727             LOG.debug("received delete GracefulRestart config val {}", val.getStalepathTime().intValue());
 728             synchronized (BgpConfigurationManager.this) {
 729                 BgpRouter br = getClient(YANG_OBJ);
 730                 if (br == null) {
 731                     LOG.error("{} Unable to delete stale-path time; {} {}", YANG_OBJ,
 732                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 733                     return;
 734                 }
 735                 try {
 736                     br.delGracefulRestart();
 737                 } catch (TException | BgpRouterException e) {
 738                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 739                 }
 740             }
 741         }
 742
 743         @Override
 744         public void update(InstanceIdentifier<GracefulRestart> iid,
 745                 GracefulRestart oldval, GracefulRestart newval) {
 746             if (!isBGPEntityOwner()) {
 747                 return;
 748             }
 749             LOG.debug("received update GracefulRestart config val {}", newval.getStalepathTime().intValue());
 750             synchronized (BgpConfigurationManager.this) {
 751                 int stalePathTime = newval.getStalepathTime().intValue();
 752                 BgpRouter br = getClient(YANG_OBJ);
 753                 if (br == null) {
 754                     LOG.error("{} Unable to update stale-path time to {}; {} {}", YANG_OBJ, stalePathTime,
 755                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 756                     return;
 757                 }
 758                 try {
 759                     br.addGracefulRestart(stalePathTime);
 760                 } catch (TException | BgpRouterException e) {
 761                     LOG.error("{} update received exception; {}", YANG_OBJ, ADD_WARN, e);
 762                 }
 763             }
 764         }
 765     }
 766
 767     public class LoggingReactor extends AbstractAsyncDataTreeChangeListener<Logging>
 768             implements ClusteredDataTreeChangeListener<Logging> {
 769
 770         private static final String YANG_OBJ = "logging ";
 771
 772         public LoggingReactor() {
 773             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 774                     InstanceIdentifier.create(Bgp.class).child(Logging.class),
 775                     org.opendaylight.infrautils.utils.concurrent.Executors
 776                             .newListeningSingleThreadExecutor("LoggingReactor", LOG));
 777         }
 778
 779         @Override
 780         public void add(InstanceIdentifier<Logging> iid, Logging val) {
 781             if (!isBGPEntityOwner()) {
 782                 return;
 783             }
 784             synchronized (BgpConfigurationManager.this) {
 785                 BgpRouter br = getClient(YANG_OBJ);
 786                 if (br == null) {
 787                     LOG.error("{} Unable to add logging for qbgp; {} {}", YANG_OBJ,
 788                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 789                     return;
 790                 }
 791                 try {
 792                     br.setLogging(val.getFile(), val.getLevel());
 793                 } catch (TException | BgpRouterException e) {
 794                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 795                 }
 796             }
 797         }
 798
 799         @Override
 800         public void remove(InstanceIdentifier<Logging> iid, Logging val) {
 801             if (!isBGPEntityOwner()) {
 802                 return;
 803             }
 804             LOG.debug("received remove Logging config val {}", val.getLevel());
 805             synchronized (BgpConfigurationManager.this) {
 806                 BgpRouter br = getClient(YANG_OBJ);
 807                 if (br == null) {
 808                     LOG.error("{} Unable to remove logging for qbgp; {} {}", YANG_OBJ,
 809                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 810                     return;
 811                 }
 812                 try {
 813                     br.setLogging(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
 814                 } catch (TException | BgpRouterException e) {
 815                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 816                 }
 817             }
 818         }
 819
 820         @Override
 821         public void update(InstanceIdentifier<Logging> iid,
 822                 Logging oldval, Logging newval) {
 823             if (!isBGPEntityOwner()) {
 824                 return;
 825             }
 826             synchronized (BgpConfigurationManager.this) {
 827                 BgpRouter br = getClient(YANG_OBJ);
 828                 if (br == null) {
 829                     LOG.error("{} Unable to update logging for qbgp; {} {}", YANG_OBJ,
 830                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 831                     return;
 832                 }
 833                 try {
 834                     br.setLogging(newval.getFile(), newval.getLevel());
 835                 } catch (TException | BgpRouterException e) {
 836                     LOG.error("{} newval received exception; {}", YANG_OBJ, ADD_WARN, e);
 837                 }
 838             }
 839         }
 840     }
 841
 842     public class NeighborsReactor extends AbstractAsyncDataTreeChangeListener<Neighbors>
 843             implements ClusteredDataTreeChangeListener<Neighbors> {
 844
 845         private static final String YANG_OBJ = "neighbors ";
 846
 847         public NeighborsReactor() {
 848             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 849                     InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class),
 850                     org.opendaylight.infrautils.utils.concurrent.Executors
 851                             .newListeningSingleThreadExecutor("NeighborsReactor", LOG));
 852         }
 853
 854         @Override
 855         public void add(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 856             if (nbrList != null && !nbrList.contains(val)) {
 857                 LOG.trace("Adding nbr {} to nbrlist", val.getAddress().getValue());
 858                 nbrList.add(val);
 859             }
 860             if (!isBGPEntityOwner()) {
 861                 return;
 862             }
 863             LOG.debug("received add Neighbors config val {}", val.getAddress().getValue());
 864             synchronized (BgpConfigurationManager.this) {
 865                 String peerIp = val.getAddress().getValue();
 866                 String sourceIp = (val.getUpdateSource() == null) ? null :
 867                                     val.getUpdateSource().getSourceIp().getValue();
 868                 int nhops = (val.getEbgpMultihop() == null) ? 0 :
 869                                     val.getEbgpMultihop().getNhops().intValue();
 870                 Map<AddressFamiliesKey, AddressFamilies> keyAddressFamiliesMap = val.getAddressFamilies();
 871                 long as = val.getRemoteAs().toJava();
 872                 final String md5Secret = extractMd5Secret(val);
 873                 BgpRouter br = getClient(YANG_OBJ);
 874                 if (br == null) {
 875                     LOG.debug("{} Unable to process add for peer {} as {}; {} {}", YANG_OBJ, peerIp, as,
 876                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 877                     return;
 878                 }
 879                 try {
 880                     //itmProvider.buildTunnelsToDCGW(new IpAddress(peerIp.toCharArray()));
 881                     br.addNeighbor(peerIp, as, md5Secret);
 882                     if (nhops != 0) {
 883                         br.addEbgpMultihop(peerIp, nhops);
 884                     }
 885                     if (sourceIp != null) {
 886                         br.addUpdateSource(peerIp, sourceIp);
 887                     }
 888                     if (keyAddressFamiliesMap != null) {
 889                         for (AddressFamilies af : keyAddressFamiliesMap.values()) {
 890                             af_afi afi = af_afi.findByValue(af.getAfi().intValue());
 891                             af_safi safi = af_safi.findByValue(af.getSafi().intValue());
 892                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
 893                         }
 894                     }
 895
 896                 } catch (TException | BgpRouterException e) {
 897                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 898                 }
 899             }
 900         }
 901
 902         @Override
 903         public void remove(InstanceIdentifier<Neighbors> iid, Neighbors val) {
 904             if (nbrList != null && nbrList.contains(val)) {
 905                 LOG.trace("Removing nbr {} from nbr list", val.getAddress().getValue());
 906                 nbrList.remove(val);
 907             }
 908             if (!isBGPEntityOwner()) {
 909                 return;
 910             }
 911             LOG.debug("received remove Neighbors config val {}", val.getAddress().getValue());
 912             synchronized (BgpConfigurationManager.this) {
 913                 String peerIp = val.getAddress().getValue();
 914                 BgpRouter br = getClient(YANG_OBJ);
 915                 if (br == null) {
 916                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 917                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 918                     return;
 919                 }
 920                 try {
 921                     //itmProvider.deleteTunnelsToDCGW(new IpAddress(val.getAddress().getValue().toCharArray()));
 922                     br.delNeighbor(peerIp);
 923                 } catch (TException | BgpRouterException e) {
 924                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
 925                 }
 926
 927                 final BgpAlarms bgpAlarms = getBgpAlarms();
 928                 if (bgpAlarms != null) {
 929                     bgpAlarms.clearBgpNbrDownAlarm(peerIp);
 930                 }
 931
 932                 if (bgpUtil.isBfdEnabled()) {
 933                     final BgpCounters bgpCounters = getBgpCounters();
 934                     if (bgpCounters != null) {
 935                         bgpCounters.clearBfdNbrCounters(peerIp);
 936                     }
 937                 }
 938             }
 939         }
 940
 941         @Override
 942         public void update(InstanceIdentifier<Neighbors> iid,
 943                 Neighbors oldval, Neighbors newval) {
 944             if (!isBGPEntityOwner()) {
 945                 return;
 946             }
 947             //purposefully nothing to do.
 948         }
 949     }
 950
 951     public class EbgpMultihopReactor extends AbstractAsyncDataTreeChangeListener<EbgpMultihop>
 952             implements ClusteredDataTreeChangeListener<EbgpMultihop> {
 953
 954         private static final String YANG_OBJ = "ebgp-multihop ";
 955
 956         public EbgpMultihopReactor() {
 957             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 958                     InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
 959                     .child(EbgpMultihop.class), org.opendaylight.infrautils.utils.concurrent.Executors
 960                             .newListeningSingleThreadExecutor("EbgpMultihopReactor", LOG));
 961         }
 962
 963         @Override
 964         public void add(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 965             if (!isBGPEntityOwner()) {
 966                 return;
 967             }
 968             LOG.debug("received add EbgpMultihop config val {}", val.getPeerIp().getValue());
 969             synchronized (BgpConfigurationManager.this) {
 970                 String peerIp = val.getPeerIp().getValue();
 971                 BgpRouter br = getClient(YANG_OBJ);
 972                 if (br == null) {
 973                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
 974                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
 975                     return;
 976                 }
 977                 try {
 978                     br.addEbgpMultihop(peerIp, val.getNhops().intValue());
 979                 } catch (TException | BgpRouterException e) {
 980                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
 981                 }
 982             }
 983         }
 984
 985         @Override
 986         public void remove(InstanceIdentifier<EbgpMultihop> iid, EbgpMultihop val) {
 987             if (!isBGPEntityOwner()) {
 988                 return;
 989             }
 990             LOG.debug("received remove EbgpMultihop config val {}", val.getPeerIp().getValue());
 991             synchronized (BgpConfigurationManager.this) {
 992                 String peerIp = val.getPeerIp().getValue();
 993                 BgpRouter br = getClient(YANG_OBJ);
 994                 if (br == null) {
 995                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
 996                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
 997                     return;
 998                 }
 999                 try {
1000                     br.delEbgpMultihop(peerIp);
1001                 } catch (TException | BgpRouterException e) {
1002                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1003                 }
1004             }
1005         }
1006
1007         @Override
1008         public void update(InstanceIdentifier<EbgpMultihop> iid,
1009                 EbgpMultihop oldval, EbgpMultihop newval) {
1010             if (!isBGPEntityOwner()) {
1011                 return;
1012             }
1013             LOG.error(YANG_OBJ + UPD_WARN);
1014         }
1015     }
1016
1017     public class UpdateSourceReactor extends AbstractAsyncDataTreeChangeListener<UpdateSource>
1018             implements ClusteredDataTreeChangeListener<UpdateSource> {
1019
1020         private static final String YANG_OBJ = "update-source ";
1021
1022         public UpdateSourceReactor() {
1023             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
1024                     InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
1025                     .child(UpdateSource.class), org.opendaylight.infrautils.utils.concurrent.Executors
1026                             .newListeningSingleThreadExecutor("UpdateSourceReactor", LOG));
1027         }
1028
1029         @Override
1030         public void add(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
1031             if (!isBGPEntityOwner()) {
1032                 return;
1033             }
1034             LOG.debug("received add UpdateSource config val {}", val.getSourceIp().getValue());
1035             synchronized (BgpConfigurationManager.this) {
1036                 String peerIp = val.getPeerIp().getValue();
1037                 BgpRouter br = getClient(YANG_OBJ);
1038                 if (br == null) {
1039                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1040                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1041                     return;
1042                 }
1043                 try {
1044                     br.addUpdateSource(peerIp, val.getSourceIp().getValue());
1045                 } catch (TException | BgpRouterException e) {
1046                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1047                 }
1048             }
1049         }
1050
1051         @Override
1052         public void remove(InstanceIdentifier<UpdateSource> iid, UpdateSource val) {
1053             if (!isBGPEntityOwner()) {
1054                 return;
1055             }
1056             LOG.debug("received remove UpdateSource config val {}", val.getSourceIp().getValue());
1057             synchronized (BgpConfigurationManager.this) {
1058                 String peerIp = val.getPeerIp().getValue();
1059                 BgpRouter br = getClient(YANG_OBJ);
1060                 if (br == null) {
1061                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1062                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1063                     return;
1064                 }
1065                 try {
1066                     br.delUpdateSource(peerIp);
1067                 } catch (TException | BgpRouterException e) {
1068                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1069                 }
1070             }
1071         }
1072
1073         @Override
1074         public void update(InstanceIdentifier<UpdateSource> iid,
1075                 UpdateSource oldval, UpdateSource newval) {
1076             if (!isBGPEntityOwner()) {
1077                 return;
1078             }
1079             LOG.error(YANG_OBJ + UPD_WARN);
1080         }
1081     }
1082
1083     public class AddressFamiliesReactor extends AbstractAsyncDataTreeChangeListener<AddressFamilies>
1084             implements ClusteredDataTreeChangeListener<AddressFamilies> {
1085
1086         private static final String YANG_OBJ = "address-families ";
1087
1088         public AddressFamiliesReactor() {
1089             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
1090                     InstanceIdentifier.create(Bgp.class).child(NeighborsContainer.class).child(Neighbors.class)
1091                     .child(AddressFamilies.class), org.opendaylight.infrautils.utils.concurrent.Executors
1092                             .newListeningSingleThreadExecutor("AddressFamiliesReactor", LOG));
1093         }
1094
1095         @Override
1096         public void add(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1097             if (!isBGPEntityOwner()) {
1098                 return;
1099             }
1100             LOG.debug("received add AddressFamilies config val {}", val.getPeerIp().getValue());
1101             synchronized (BgpConfigurationManager.this) {
1102                 String peerIp = val.getPeerIp().getValue();
1103                 BgpRouter br = getClient(YANG_OBJ);
1104                 if (br == null) {
1105                     LOG.debug("{} Unable to process add for peer {}; {} {}", YANG_OBJ, peerIp,
1106                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1107                     return;
1108                 }
1109                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1110                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1111                 try {
1112                     br.addAddressFamily(peerIp, afi, safi);
1113                 } catch (TException | BgpRouterException e) {
1114                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1115                 }
1116             }
1117         }
1118
1119         @Override
1120         public void remove(InstanceIdentifier<AddressFamilies> iid, AddressFamilies val) {
1121             if (!isBGPEntityOwner()) {
1122                 return;
1123             }
1124             LOG.debug("received remove AddressFamilies config val {}", val.getPeerIp().getValue());
1125             synchronized (BgpConfigurationManager.this) {
1126                 String peerIp = val.getPeerIp().getValue();
1127                 BgpRouter br = getClient(YANG_OBJ);
1128                 if (br == null) {
1129                     LOG.debug("{} Unable to process remove for peer {}; {} {}", YANG_OBJ, peerIp,
1130                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1131                     return;
1132                 }
1133                 af_afi afi = af_afi.findByValue(val.getAfi().intValue());
1134                 af_safi safi = af_safi.findByValue(val.getSafi().intValue());
1135                 try {
1136                     br.delAddressFamily(peerIp, afi, safi);
1137                 } catch (TException | BgpRouterException e) {
1138                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1139                 }
1140             }
1141         }
1142
1143         @Override
1144         public void update(InstanceIdentifier<AddressFamilies> iid,
1145                 AddressFamilies oldval, AddressFamilies newval) {
1146             if (!isBGPEntityOwner()) {
1147                 return;
1148             }
1149             LOG.error(YANG_OBJ + UPD_WARN);
1150         }
1151     }
1152
1153     public class NetworksReactor extends AbstractAsyncDataTreeChangeListener<Networks>
1154             implements ClusteredDataTreeChangeListener<Networks> {
1155
1156         private static final String YANG_OBJ = "networks ";
1157
1158         public NetworksReactor() {
1159             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
1160                     InstanceIdentifier.create(Bgp.class).child(NetworksContainer.class).child(Networks.class),
1161                     org.opendaylight.infrautils.utils.concurrent.Executors
1162                             .newListeningSingleThreadExecutor("NetworksReactor", LOG));
1163         }
1164
1165         public void add(InstanceIdentifier<Networks> iid, Networks val) {
1166             if (!isBGPEntityOwner()) {
1167                 return;
1168             }
1169             LOG.debug("received add Networks config val {}", val.getPrefixLen());
1170             synchronized (BgpConfigurationManager.this) {
1171                 String rd = val.getRd();
1172                 String pfxlen = val.getPrefixLen();
1173                 String nh = val.getNexthop().getValue();
1174                 BgpRouter br = getClient(YANG_OBJ);
1175                 if (br == null) {
1176                     LOG.debug("{} Unable to process add for rd {} prefix {} nexthop {}; {} {}", YANG_OBJ, rd, pfxlen,
1177                             nh, BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1178                     return;
1179                 }
1180                 Long label = val.getLabel().toJava();
1181                 int lbl = label == null ? qbgpConstants.LBL_NO_LABEL
1182                         : label.intValue();
1183                 int l3vni = val.getL3vni() == null ? qbgpConstants.LBL_NO_LABEL
1184                         : val.getL3vni().intValue();
1185                 int l2vni = val.getL2vni() == null ? qbgpConstants.LBL_NO_LABEL
1186                         : val.getL2vni().intValue();
1187
1188                 BgpControlPlaneType protocolType = val.getBgpControlPlaneType();
1189                 int ethernetTag = val.getEthtag().intValue();
1190                 String esi = val.getEsi();
1191                 String macaddress = val.getMacaddress();
1192                 EncapType encapType = val.getEncapType();
1193                 String routerMac = val.getRoutermac();
1194
1195                 try {
1196                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni, BgpUtil.convertToThriftProtocolType(protocolType),
1197                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
1198                 } catch (TException | BgpRouterException e) {
1199                     LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1200                 }
1201             }
1202         }
1203
1204         @Override
1205         public void remove(InstanceIdentifier<Networks> iid, Networks val) {
1206             if (!isBGPEntityOwner()) {
1207                 return;
1208             }
1209             LOG.debug("received remove Networks config val {}", val.getPrefixLen());
1210             synchronized (BgpConfigurationManager.this) {
1211                 String rd = val.getRd();
1212                 String pfxlen = val.getPrefixLen();
1213                 BgpRouter br = getClient(YANG_OBJ);
1214                 if (br == null) {
1215                     LOG.debug("{} Unable to process remove for rd {} prefix {}; {} {}", YANG_OBJ, rd, pfxlen,
1216                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1217                     return;
1218                 }
1219                 Long label = val.getLabel().toJava();
1220                 int lbl = label == null ? 0 : label.intValue();
1221                 if (rd == null && lbl > 0) {
1222                     //LU prefix is being deleted.
1223                     rd = Integer.toString(lbl);
1224                 }
1225                 try {
1226                     br.delPrefix(rd, pfxlen);
1227                 } catch (TException | BgpRouterException e) {
1228                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1229                 }
1230             }
1231         }
1232
1233         /**get the value AFI from a prefix as "x.x.x.x/x".
1234          *
1235          * @param pfxlen the prefix to get an afi
1236          * @return the afi value as you are need
1237          */
1238         public  int  testValueAFI(String pfxlen) {
1239             int afiNew = af_afi.AFI_IP.getValue();
1240             try {
1241                 String ipOnly = pfxlen.substring(0, pfxlen.lastIndexOf("/"));
1242                 java.net.Inet6Address.getByName(ipOnly);
1243                 afiNew = af_afi.AFI_IPV6.getValue();
1244             } catch (java.net.UnknownHostException e) {
1245                 //ce n'est pas de l'ipv6
1246             }
1247             return afiNew;
1248         }
1249
1250
1251         @Override
1252         public void update(final InstanceIdentifier<Networks> iid,
1253                 final Networks oldval, final Networks newval) {
1254             if (!isBGPEntityOwner()) {
1255                 return;
1256             }
1257             if (oldval.equals(newval)) {
1258                 //Update: OLD and New values are same, no need to trigger remove/add.
1259                 LOG.debug("received Updated for the same OLD and New values. RD: {}, Prefix: {}, Label: {}, NH: {}",
1260                         oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop());
1261                 return;
1262             }
1263             LOG.debug("update networks old val RD: {}, Prefix: {}, Label: {}, NH: {} "
1264                             + "new val RD: {}, Prefix: {}, Label: {}, NH: {}",
1265                     oldval.getRd(), oldval.getPrefixLen(), oldval.getLabel(), oldval.getNexthop(),
1266                     newval.getRd(), newval.getPrefixLen(), newval.getLabel(), newval.getNexthop());
1267             remove(iid, oldval);
1268             add(iid, newval);
1269         }
1270     }
1271
1272     static Timer timer = new Timer();
1273
1274     public class VrfsReactor extends AbstractAsyncDataTreeChangeListener<Vrfs>
1275             implements ClusteredDataTreeChangeListener<Vrfs> {
1276
1277         private static final String YANG_OBJ = "vrfs ";
1278
1279         public VrfsReactor() {
1280             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
1281                     InstanceIdentifier.create(Bgp.class).child(VrfsContainer.class).child(Vrfs.class),
1282                     org.opendaylight.infrautils.utils.concurrent.Executors
1283                             .newListeningSingleThreadExecutor("VrfsReactor", LOG));
1284         }
1285
1286         @Override
1287         public void add(InstanceIdentifier<Vrfs> iid, Vrfs vrfs) {
1288             if (!isBGPEntityOwner()) {
1289                 return;
1290             }
1291             LOG.debug("received add Vrfs config value {}", vrfs.getRd());
1292             synchronized (BgpConfigurationManager.this) {
1293                 String rd = vrfs.getRd();
1294                 BgpRouter br = getClient(YANG_OBJ);
1295                 if (br == null) {
1296                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1297                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1298                     return;
1299                 }
1300                 try {
1301                     Map<AddressFamiliesVrfKey, AddressFamiliesVrf> keyAddressFamiliesVrfMap
1302                             = vrfs.getAddressFamiliesVrf();
1303                     for (AddressFamiliesVrf vrfAddrFamily : keyAddressFamiliesVrfMap.values()) {
1304                         /*add to br the new vrfs arguments*/
1305                         br.addVrf(BgpUtil.getLayerType(vrfAddrFamily), rd, vrfs.getImportRts(),
1306                                 vrfs.getExportRts(), vrfAddrFamily.getAfi().toJava(), vrfAddrFamily.getSafi().toJava());
1307                     }
1308                     /*add to br the vrfs contained in mapNewAdFamily*/
1309                     List<AddressFamiliesVrf> vrfAddrFamilyListFromMap = mapNewAdFamily.get(rd);
1310                     if (vrfAddrFamilyListFromMap == null) {
1311                         return;
1312                     }
1313
1314                     for (AddressFamiliesVrf adf : vrfAddrFamilyListFromMap) {
1315                         if (keyAddressFamiliesVrfMap.values().contains(adf)) {
1316                             mapNewAdFamily.remove(rd);
1317                         } else  if (adf != null) {
1318
1319                             br.addVrf(BgpUtil.getLayerType(adf), rd, vrfs.getImportRts(),
1320                                     vrfs.getExportRts(), adf.getAfi().toJava(), adf.getSafi().toJava());
1321                             // remove AddressFamiliesVrf which was already added to BGP
1322                             vrfAddrFamilyListFromMap.remove(adf);
1323                             if (vrfAddrFamilyListFromMap.isEmpty()) {
1324                                 // remove Vrf entry from temp mapNewAdFamily if all its AddressFamiliesVrf was
1325                                 // added to BGP
1326                                 mapNewAdFamily.remove(rd);
1327                             }
1328                         }
1329                     }
1330                 } catch (TException | BgpRouterException e) {
1331                     LOG.error("{} get {}, Add received exception", YANG_OBJ, ADD_WARN, e);
1332                 }
1333             }
1334         }
1335
1336         @Override
1337         public void remove(InstanceIdentifier<Vrfs> iid, Vrfs val) {
1338             if (!isBGPEntityOwner()) {
1339                 return;
1340             }
1341             LOG.debug("received remove Vrfs config val {}", val.getRd());
1342             synchronized (BgpConfigurationManager.this) {
1343                 String rd = val.getRd();
1344                 BgpRouter br = getClient(YANG_OBJ);
1345                 if (br == null) {
1346                     LOG.debug("{} Unable to process remove for rd {}; {} {}", YANG_OBJ, rd,
1347                             BgpRouterException.BGP_ERR_NOT_INITED, DEL_WARN);
1348                     return;
1349                 }
1350                 try {
1351                     List<AddressFamiliesVrf> adf = mapNewAdFamily.get(rd);
1352                     adf = adf != null ? adf : new ArrayList<>();
1353                     for (AddressFamiliesVrf s : val.getAddressFamiliesVrf().values()) {
1354                         br.delVrf(rd, s.getAfi().toJava(), s.getSafi().toJava());
1355                         adf.remove(s);// remove in the map the vrf in waiting for advertise quagga
1356                     }
1357                     if (adf.isEmpty()) {
1358                         mapNewAdFamily.remove(rd);
1359                     }
1360                 } catch (TException | BgpRouterException e) {
1361                     LOG.error("{} Delete received exception; {}", YANG_OBJ, DEL_WARN, e);
1362                 }
1363             }
1364         }
1365
1366         @Override
1367         public void update(InstanceIdentifier<Vrfs> iid,
1368                 Vrfs oldval, Vrfs newval) {
1369             if (oldval != null && newval != null) {
1370                 LOG.debug("received update Vrfs config val {}, VRFS: Update getting triggered for VRFS rd {}",
1371                         newval.getRd(), oldval.getRd());
1372             } else {
1373                 LOG.debug("received update Vrfs config val {}, from old vrf {}",
1374                         newval, oldval);
1375             }
1376             if (!isBGPEntityOwner()) {
1377                 return;
1378             }
1379
1380             List<AddressFamiliesVrf> adFamilyVrfToDel = new ArrayList<>();
1381             List<AddressFamiliesVrf> adFamilyVrfToAdd = new ArrayList<>();
1382             List<AddressFamiliesVrf> oldlistAdFamilies = new ArrayList<>();
1383             List<AddressFamiliesVrf> newlistAdFamilies = new ArrayList<>();
1384             if (oldval != null) {
1385                 oldlistAdFamilies = oldval.getAddressFamiliesVrf() == null
1386                         ? new ArrayList<>()
1387                         : new ArrayList<AddressFamiliesVrf>(oldval.getAddressFamiliesVrf().values());
1388             }
1389             if (newval != null) {
1390                 newlistAdFamilies = newval.getAddressFamiliesVrf() == null
1391                         ? new ArrayList<>()
1392                         : new ArrayList<AddressFamiliesVrf>(newval.getAddressFamiliesVrf().values());
1393             }
1394             /*find old AddressFamily to remove from new configuration*/
1395             for (AddressFamiliesVrf adVrf : oldlistAdFamilies) {
1396                 if (!newlistAdFamilies.contains(adVrf)) {
1397                     adFamilyVrfToDel.add(adVrf);
1398                 }
1399             }
1400             /*find new AddressFamily to add to unexisting configuration*/
1401             for (AddressFamiliesVrf adVrf : newlistAdFamilies) {
1402                 if (!oldlistAdFamilies.contains(adVrf)) {
1403                     adFamilyVrfToAdd.add(adVrf);
1404                 }
1405             }
1406             String rd = newval != null ? newval.getRd() : null;
1407             if (rd != null) {
1408                 BgpRouter br = getClient(YANG_OBJ);
1409                 if (br == null) {
1410                     LOG.debug("{} Unable to process add for rd {}; {} {}", YANG_OBJ, rd,
1411                             BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1412                     return;
1413                 }
1414
1415                 for (AddressFamiliesVrf adfvrf : adFamilyVrfToAdd) {
1416                     try {
1417                         LOG.debug("call addVRf rd {} afi {} safi {}", rd, adfvrf.getAfi(), adfvrf.getSafi());
1418                         br.addVrf(BgpUtil.getLayerType(adfvrf), rd, newval.getImportRts(),
1419                                 newval.getExportRts(),adfvrf.getAfi().toJava(), adfvrf.getSafi().toJava());
1420                     } catch (TException | BgpRouterException e) {
1421                         LOG.error("{} Add received exception; {}", YANG_OBJ, ADD_WARN, e);
1422                     }
1423                 }
1424
1425                 for (AddressFamiliesVrf adfToDel : adFamilyVrfToDel) {
1426                     try {
1427                         LOG.debug("call delVRf rd {} afi {} safi {}", rd, adfToDel.getAfi(), adfToDel.getSafi());
1428                         br.delVrf(rd, adfToDel.getAfi().toJava(), adfToDel.getSafi().toJava());
1429                     } catch (TException | BgpRouterException e) {
1430                         LOG.error("{} delVrf received exception; {}", YANG_OBJ, ADD_WARN, e);
1431                     }
1432                 }
1433             }
1434         }
1435     }
1436
1437     public class BgpReactor extends AbstractAsyncDataTreeChangeListener<Bgp>
1438             implements ClusteredDataTreeChangeListener<Bgp> {
1439
1440         private static final String YANG_OBJ = "Bgp ";
1441
1442         public BgpReactor() {
1443             super(dataBroker, LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(Bgp.class),
1444                     org.opendaylight.infrautils.utils.concurrent.Executors
1445                             .newListeningSingleThreadExecutor("BgpReactor", LOG));
1446         }
1447
1448         @Override
1449         public void add(InstanceIdentifier<Bgp> iid, Bgp val) {
1450             LOG.debug("received add Bgp config");
1451
1452             try {
1453                 initer.await();
1454             } catch (InterruptedException e) {
1455                 // Ignored
1456             }
1457             synchronized (BgpConfigurationManager.this) {
1458                 config = val;
1459                 if (!isBGPEntityOwner()) {
1460                     return;
1461                 }
1462             }
1463         }
1464
1465         @Override
1466         public void remove(InstanceIdentifier<Bgp> iid, Bgp val) {
1467             if (!isBGPEntityOwner()) {
1468                 return;
1469             }
1470             LOG.debug("received remove Bgp config");
1471
1472             config = null;
1473         }
1474
1475         @Override
1476         public void update(InstanceIdentifier<Bgp> iid,
1477                 Bgp oldval, Bgp newval) {
1478             if (!isBGPEntityOwner()) {
1479                 return;
1480             }
1481
1482             config = newval;
1483         }
1484     }
1485
1486     @SuppressWarnings("deprecation")
1487     public class MultipathReactor extends AbstractAsyncDataTreeChangeListener<Multipath>
1488             implements ClusteredDataTreeChangeListener<Multipath> {
1489
1490         private static final String YANG_OBJ = "multipath ";
1491
1492         public MultipathReactor() {
1493             super(dataBroker, LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(Bgp.class)
1494                     .child(MultipathContainer.class).child(Multipath.class),
1495                     org.opendaylight.infrautils.utils.concurrent.Executors
1496                             .newListeningSingleThreadExecutor("MultipathReactor", LOG));
1497         }
1498
1499         @Override
1500         public void remove(InstanceIdentifier<Multipath> iid, Multipath val) {
1501             executor.execute(new MultipathStatusChange(val));
1502         }
1503
1504         @Override
1505         public void update(InstanceIdentifier<Multipath> iid, Multipath oldval, Multipath newval) {
1506             executor.execute(new MultipathStatusChange(newval));
1507         }
1508
1509         @Override
1510         public void add(InstanceIdentifier<Multipath> key, Multipath dataObjectModification) {
1511             executor.execute(new MultipathStatusChange(dataObjectModification));
1512         }
1513
1514         class MultipathStatusChange implements Runnable {
1515
1516             Multipath multipath;
1517
1518             MultipathStatusChange(Multipath multipath) {
1519                 this.multipath = multipath;
1520             }
1521
1522             @Override
1523             public void run() {
1524                 if (isBGPEntityOwner()) {
1525                     synchronized (BgpConfigurationManager.this) {
1526
1527                         BgpRouter br = getClient(YANG_OBJ);
1528
1529                         if (br != null) {
1530                             af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
1531                             af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
1532
1533                             try {
1534                                 if (multipath.isMultipathEnabled()) {
1535                                     br.enableMultipath(afi, safi);
1536                                 } else {
1537                                     br.disableMultipath(afi, safi);
1538                                 }
1539                             } catch (TException | BgpRouterException e) {
1540                                 LOG.error("{} received exception", YANG_OBJ, e);
1541                             }
1542                         }
1543                     }
1544                 }
1545             }
1546
1547         }
1548
1549         @Override
1550         public void close() {
1551             super.close();
1552         }
1553     }
1554
1555     @SuppressWarnings("deprecation")
1556     public class VrfMaxpathReactor extends AbstractAsyncDataTreeChangeListener<VrfMaxpath>
1557             implements ClusteredDataTreeChangeListener<VrfMaxpath> {
1558
1559         private static final String YANG_OBJ = "vrfMaxpath ";
1560
1561         public VrfMaxpathReactor() {
1562             super(dataBroker, LogicalDatastoreType.CONFIGURATION,
1563                     InstanceIdentifier.create(Bgp.class).child(VrfMaxpathContainer.class).child(VrfMaxpath.class),
1564                     org.opendaylight.infrautils.utils.concurrent.Executors
1565                             .newListeningSingleThreadExecutor("VrfMaxpathReactor", LOG));
1566         }
1567
1568         class VrfMaxPathConfigurator implements Runnable {
1569
1570             VrfMaxpath vrfMaxpathVal;
1571
1572             VrfMaxPathConfigurator(VrfMaxpath vrfMaxPathVal) {
1573                 this.vrfMaxpathVal = vrfMaxPathVal;
1574             }
1575
1576             @Override
1577             public void run() {
1578                 if (isBGPEntityOwner()) {
1579                     synchronized (BgpConfigurationManager.this) {
1580                         BgpRouter br = getClient(YANG_OBJ);
1581                         if (br != null) {
1582                             try {
1583                                 br.multipaths(vrfMaxpathVal.getRd(), vrfMaxpathVal.getMaxpaths().toJava());
1584                                 LOG.debug("Maxpath for vrf {} is {}", vrfMaxpathVal.getRd(),
1585                                         vrfMaxpathVal.getMaxpaths());
1586                             } catch (TException | BgpRouterException e) {
1587                                 LOG.error("{} received exception", YANG_OBJ, e);
1588                             }
1589                         }
1590                     }
1591                 }
1592             }
1593         }
1594
1595         @Override
1596         public void remove(InstanceIdentifier<VrfMaxpath> iid, VrfMaxpath vrfMaxPathVal) {
1597             if (isBGPEntityOwner()) {
1598                 synchronized (BgpConfigurationManager.this) {
1599                     BgpRouter br = getClient(YANG_OBJ);
1600                     if (br != null) {
1601                         try {
1602                             br.multipaths(vrfMaxPathVal.getRd(), BgpConstants.BGP_DEFAULT_MULTIPATH);
1603                             LOG.debug("Del Maxpath for vrf: {} ", vrfMaxPathVal.getRd());
1604                         } catch (TException | BgpRouterException e) {
1605                             LOG.error(YANG_OBJ + " del received exception:", e);
1606                         }
1607                     }
1608                 }
1609             }
1610         }
1611
1612         @Override
1613         public void update(InstanceIdentifier<VrfMaxpath> iid,
1614                               VrfMaxpath oldval, VrfMaxpath newval) {
1615             if (!Objects.equals(oldval.getMaxpaths(), newval.getMaxpaths())) {
1616                 executor.execute(new VrfMaxPathConfigurator(newval));
1617             }
1618         }
1619
1620         @Override
1621         public void add(InstanceIdentifier<VrfMaxpath> instanceIdentifier, VrfMaxpath vrfMaxpathVal) {
1622             executor.execute(new VrfMaxPathConfigurator(vrfMaxpathVal));
1623         }
1624
1625         @Override
1626         public void close() {
1627             super.close();
1628         }
1629     }
1630
1631     public class BfdConfigReactor extends AbstractAsyncDataTreeChangeListener<BfdConfig>
1632             implements ClusteredDataTreeChangeListener<BfdConfig> {
1633
1634         private static final String YANG_OBJ = "BfdConfig ";
1635
1636         public BfdConfigReactor() {
1637             super(dataBroker, LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(BfdConfig.class),
1638                     org.opendaylight.infrautils.utils.concurrent.Executors
1639                             .newListeningSingleThreadExecutor("BfdConfigReactor", LOG));
1640         }
1641
1642         @Override
1643         public void add(InstanceIdentifier<BfdConfig> iid, BfdConfig val) {
1644             if (!isBGPEntityOwner()) {
1645                 return;
1646             }
1647             BgpRouter br = getClient(YANG_OBJ);
1648             LOG.debug("received bfd config: bfd enabled {} min-rx {} min-tx {} detect-mul {} mhop {}",
1649                     val.isBfdEnabled(), val.getMinRx(), val.getMinTx(),
1650                     val.getDetectMult(), val.isMultihop());
1651             if (br == null) {
1652                 LOG.debug(YANG_OBJ + "{} Unable to process add  {}",
1653                         BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1654                 return;
1655             }
1656             if (val.isBfdEnabled() == false) {
1657                 LOG.debug("BFD not enabled. Ignoring the config add");
1658                 return;
1659             }
1660             int minRx = val.getMinRx().intValue();
1661             int minTx = val.getMinTx().intValue();
1662             int detectMult = val.getDetectMult().intValue();
1663             boolean multiHop = val.isMultihop();
1664             try {
1665                 br.addBfd(detectMult, minRx, minTx,multiHop);
1666             } catch (TException | BgpRouterException e) {
1667                 LOG.error("{} get {}, Add received exception;", YANG_OBJ, ADD_WARN, e);
1668             }
1669         }
1670
1671         @Override
1672         public void remove(InstanceIdentifier<BfdConfig> iid, BfdConfig val) {
1673             if (!isBGPEntityOwner()) {
1674                 return;
1675             }
1676             LOG.debug("received bfd config removal");
1677             BgpRouter br = getClient(YANG_OBJ);
1678             if (br == null) {
1679                 LOG.debug("{} Unable to process del {}  {}", YANG_OBJ,
1680                         BgpRouterException.BGP_ERR_NOT_INITED, ADD_WARN);
1681                 return;
1682             }
1683             try {
1684                 br.delBfd();
1685             } catch (TException | BgpRouterException e) {
1686                 LOG.error("{} get {}, Del received exception;", YANG_OBJ, ADD_WARN, e);
1687             }
1688
1689         }
1690
1691         @Override
1692         public void update(InstanceIdentifier<BfdConfig> iid,
1693                               BfdConfig oldval, BfdConfig newval) {
1694             LOG.debug("received bfd config: updated oldval bfd enabled {}"
1695                     + "min-rx {} min-tx {} detect-mul {} mhop {}",
1696                     oldval.isBfdEnabled(), oldval.getMinRx(), oldval.getMinTx(),
1697                     oldval.getDetectMult(), oldval.isMultihop());
1698             LOG.debug("received bfd config: updated newval bfd enabled {}"
1699                     + "min-rx {} min-tx {} detect-mul {} mhop {}",
1700                     newval.isBfdEnabled(), newval.getMinRx(), newval.getMinTx(),
1701                     newval.getDetectMult(), newval.isMultihop());
1702             if (oldval.isBfdEnabled()) {
1703                 LOG.debug("deleting bfd config on an update");
1704                 remove(iid, oldval);
1705             }
1706             LOG.debug("adding bfd config on an update");
1707             add(iid, newval);
1708         }
1709     }
1710
1711
1712     public boolean isIpAvailable(String odlip) {
1713
1714         try {
1715             if (odlip != null) {
1716                 if ("127.0.0.1".equals(odlip)) {
1717                     return true;
1718                 }
1719                 Enumeration<NetworkInterface> networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces();
1720                 while (networkInterfaceEnumeration.hasMoreElements()) {
1721                     NetworkInterface networkInterface = networkInterfaceEnumeration.nextElement();
1722                     Enumeration<InetAddress>  inetAddressEnumeration = networkInterface.getInetAddresses();
1723                     while (inetAddressEnumeration.hasMoreElements()) {
1724                         InetAddress inetAddress = inetAddressEnumeration.nextElement();
1725                         if (odlip.equals(inetAddress.getHostAddress())) {
1726                             return true;
1727                         }
1728                     }
1729                 }
1730             }
1731         } catch (SocketException e) {
1732             // Ignored?
1733         }
1734         return false;
1735     }
1736
1737     public long getStalePathtime(int defValue, AsId asId) {
1738         long spt = 0;
1739         try {
1740             InstanceIdentifier<GracefulRestart> id =
1741                     InstanceIdentifier.create(Bgp.class).child(GracefulRestart.class);
1742             Optional<GracefulRestart> gracefulRestartOptional = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1743                     LogicalDatastoreType.CONFIGURATION, id);
1744             if (gracefulRestartOptional.isPresent()) {
1745                 spt = gracefulRestartOptional.get().getStalepathTime().toJava();
1746             }
1747         } catch (NullPointerException e) {
1748             try {
1749                 spt = asId.getStalepathTime().toJava();
1750                 LOG.trace("BGP config/Stale-path time is not set using graceful");
1751             } catch (NullPointerException ignore) {
1752                 LOG.trace("BGP AS id is not set using graceful");
1753                 spt = defValue;
1754             }
1755         } catch (InterruptedException | ExecutionException e) {
1756             LOG.trace("Exception while reading GracefulRestart DS for the As {}", asId.getStalepathTime());
1757             spt = defValue;
1758         }
1759         if (spt == 0) {
1760             LOG.trace("BGP config/Stale-path time is not set using graceful/start-bgp");
1761             spt = defValue;
1762         }
1763         return spt;
1764     }
1765
1766     public static boolean isValidConfigBgpHostPort(String bgpHost, int bgpPort) {
1767         if (!bgpHost.equals(DEF_CHOST)) {
1768             return true;
1769         } else {
1770             return false;
1771         }
1772     }
1773
1774     public synchronized void bgpRestarted() {
1775         /*
1776          * If there a thread which in the process of stale cleanup, cancel it
1777          * and start a new thread (to avoid processing same again).
1778          */
1779         if (previousReplayJobInProgress()) {
1780             cancelPreviousReplayJob();
1781         }
1782         Runnable task = () -> {
1783             try {
1784                 LOG.info("running bgp replay task ");
1785                 if (get() == null) {
1786                     String host = getConfigHost();
1787                     int port = getConfigPort();
1788                     LOG.info("connecting  to bgp host {} ", host);
1789                     bgpRouter.connect(host, port);
1790                     LOG.info("no config to push in bgp replay task ");
1791                     return;
1792                 }
1793                 setStaleStartTime(System.currentTimeMillis());
1794                 LOG.info("started creating stale fibDSWriter  map ");
1795                 createStaleFibMap();
1796                 setStaleEndTime(System.currentTimeMillis());
1797                 LOG.info("took {} msecs for stale fibDSWriter map creation ", getStaleEndTime() - getStaleStartTime());
1798                 LOG.info("started bgp config replay ");
1799                 setCfgReplayStartTime(System.currentTimeMillis());
1800                 boolean replaySucceded = replay();
1801                 setCfgReplayEndTime(System.currentTimeMillis());
1802                 LOG.info("took {} msecs for bgp replay ", getCfgReplayEndTime() - getCfgReplayStartTime());
1803                 if (replaySucceded) {
1804                     long routeSyncTime = getStalePathtime(BGP_RESTART_ROUTE_SYNC_SEC, config.getAsId());
1805                     setStaleCleanupTime(routeSyncTime);
1806                     LOG.error("starting the stale cleanup timer: {} seconds", routeSyncTime);
1807                     routeCleanupFuture = executor.schedule(new RouteCleanup(), routeSyncTime, TimeUnit.SECONDS);
1808                 } else {
1809                     LOG.error("skipping stale cleanup, may be due to exception while replay");
1810                     staledFibEntriesMap.clear();
1811                 }
1812             } catch (InterruptedException | TimeoutException | ExecutionException eCancel) {
1813                 LOG.error("Stale Cleanup Task Cancelled", eCancel);
1814             }
1815         };
1816         lastReplayJobFt = executor.submit(task);
1817     }
1818
1819     private boolean previousReplayJobInProgress() {
1820         return ((lastReplayJobFt != null && !lastReplayJobFt.isDone())
1821                 || (routeCleanupFuture != null && !routeCleanupFuture.isDone()));
1822     }
1823
1824     private void cancelPreviousReplayJob() {
1825         try {
1826             LOG.error("cancelling already running bgp replay task");
1827             if (lastReplayJobFt != null) {
1828                 lastReplayJobFt.cancel(true);
1829                 lastReplayJobFt = null;
1830                 staledFibEntriesMap.clear();
1831             }
1832             if (routeCleanupFuture != null) {
1833                 routeCleanupFuture.cancel(true);
1834                 routeCleanupFuture = null;
1835                 staledFibEntriesMap.clear();
1836             }
1837             Thread.sleep(2000);
1838         } catch (InterruptedException e) {
1839             LOG.error("Failed to cancel previous replay job ", e);
1840         }
1841     }
1842
1843     private void doRouteSync() {
1844         for (af_afi afi : af_afi.values()) {
1845             try {
1846                 bgpRouter.initRibSync(bgpSyncHandle);
1847             } catch (BgpRouterException e) {
1848                 LOG.error("Route sync aborted, exception when initializing", e);
1849                 return;
1850             }
1851             LOG.error("Starting BGP route sync for afi {}", afi.getValue());
1852             while (bgpSyncHandle.getState() != BgpSyncHandle.DONE) {
1853                 Routes routes = null;
1854                 int noUpdates = 0;
1855                 try {
1856                     routes = bgpRouter.doRibSync(bgpSyncHandle, afi);
1857                 } catch (TException | BgpRouterException e) {
1858                     LOG.error("Route sync aborted, exception when syncing", e);
1859                     return;
1860                 }
1861                 Iterator<Update> updates = routes.getUpdatesIterator();
1862                 while (updates.hasNext()) {
1863                     noUpdates++;
1864                     Update update = updates.next();
1865                     String rd = update.getRd();
1866                     String nexthop = update.getNexthop();
1867
1868                     // TODO: decide correct label here
1869                     int label = update.getL3label();
1870                     int l2label = update.getL2label();
1871
1872                     String prefix = update.getPrefix();
1873                     int plen = update.getPrefixlen();
1874
1875
1876                     // TODO: protocol type will not be available in "update"
1877                     // use "rd" to query vrf table and obtain the protocol_type.
1878                     // Currently using PROTOCOL_EVPN as default.
1879                     onUpdatePushRoute(
1880                            protocol_type.PROTOCOL_L3VPN,
1881                            rd,
1882                            prefix,
1883                            plen,
1884                            nexthop,
1885                            update.getMacaddress(),
1886                            Uint32.valueOf(label),
1887                            Uint32.valueOf(l2label),
1888                            update.getRoutermac(),
1889                            afi);
1890                 }
1891                 LOG.error("No of updates for afi {} is {}", afi.getValue(), noUpdates);
1892             }
1893         }
1894         try {
1895             LOG.error("Ending BGP route-sync");
1896             bgpRouter.endRibSync(bgpSyncHandle);
1897         } catch (BgpRouterException e) {
1898             LOG.error("Route sync aborted, exception when ending", e);
1899         }
1900     }
1901
1902     public void addTepToElanDS(String rd, String tepIp, String mac, Uint32 l2vni) {
1903         boolean needUpdate = addToRt2TepMap(rd, tepIp, mac, l2vni);
1904         if (needUpdate) {
1905             LOG.info("Adding tepIp {} with RD {} to ELan DS", tepIp, rd);
1906             bgpUtil.addTepToElanInstance(rd, tepIp);
1907         } else {
1908             LOG.debug("Skipping the Elan update for RT2 from tep {} rd {}", tepIp, rd);
1909         }
1910     }
1911
1912     public void deleteTepfromElanDS(String rd, String tepIp, String mac) {
1913         boolean needUpdate = deleteFromRt2TepMap(rd, tepIp, mac);
1914         if (needUpdate) {
1915             LOG.info("Deleting tepIp {} with RD {} to ELan DS", tepIp, rd);
1916             bgpUtil.deleteTepFromElanInstance(rd, tepIp);
1917         } else {
1918             LOG.debug("Skipping the Elan update for RT2 withdraw from tep {} rd {}", tepIp, rd);
1919         }
1920     }
1921
1922     /* onUpdatePushRoute
1923      * Get Stale fibDSWriter map, and compare current route/fibDSWriter entry.
1924      *  - Entry compare shall include NextHop, Label.
1925      *  - If entry matches: delete from STALE Map. NO Change to FIB Config DS.
1926      *  - If entry not found, add to FIB Config DS.
1927      *  - If entry found, but either Label/NextHop doesn't match.
1928      *      - Update FIB Config DS with modified values.
1929      *      - delete from Stale Map.
1930      */
1931
1932     public void onUpdatePushRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
1933                                   String macaddress, Uint32 label, Uint32 l2label, String routermac, af_afi afi) {
1934         PrefixUpdateEvent prefixUpdateEvent = new PrefixUpdateEvent(protocolType,rd,prefix,plen,nextHop,
1935                 macaddress,label,l2label,routermac,afi);
1936         bgpUpdatesHistory.addToHistory(TransactionType.ADD, prefixUpdateEvent);
1937         boolean addroute = false;
1938         boolean macupdate = false;
1939         Uint32 l3vni = Uint32.ZERO;
1940         VrfEntry.EncapType encapType = VrfEntry.EncapType.Mplsgre;
1941         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
1942             encapType = VrfEntry.EncapType.Vxlan;
1943             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
1944             if (vpnInstanceOpDataEntry != null) {
1945                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
1946                     LOG.info("Got RT2 route for RD {} l3label {} l2label {} from tep {} with mac {} remote RD {}",
1947                             vpnInstanceOpDataEntry.getVpnInstanceName(), label, l2label, nextHop, macaddress, rd);
1948                     addTepToElanDS(rd, nextHop, macaddress, l2label);
1949                     macupdate = true;
1950                 } else {
1951                     l3vni = vpnInstanceOpDataEntry.getL3vni();
1952                 }
1953             } else {
1954                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
1955                 return;
1956             }
1957         }
1958
1959         if (!staledFibEntriesMap.isEmpty()) {
1960             // restart Scenario, as MAP is not empty.
1961             Map<String, Uint32> map = staledFibEntriesMap.get(rd);
1962             if (map != null) {
1963                 String prefixNextHop = appendNextHopToPrefix(prefix + "/" + plen, nextHop);
1964                 Uint32 labelInStaleMap = map.get(prefixNextHop);
1965                 if (null == labelInStaleMap) {
1966                     // New Entry, which happened to be added during restart.
1967                     addroute = true;
1968                 } else {
1969                     map.remove(prefixNextHop);
1970                     if (isRouteModified(label, labelInStaleMap)) {
1971                         LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1972                         // Existing entry, where in Label got modified during restart
1973                         addroute = true;
1974                     }
1975                 }
1976             } else {
1977                 LOG.debug("rd {} map is null while processing prefix {} ", rd, prefix);
1978                 addroute = true;
1979             }
1980         } else {
1981             LOG.debug("Route add ** {} ** {}/{} ** {} ** {} ", rd, prefix, plen, nextHop, label);
1982             addroute = true;
1983         }
1984         if (macupdate) {
1985             LOG.info("ADD: Adding Mac Fib entry rd {} mac{} nexthop {} l2vni {}", rd, macaddress, nextHop, l2label);
1986             fibDSWriter.addMacEntryToDS(rd, macaddress, prefix, Collections.singletonList(nextHop),
1987                     encapType, l2label, routermac, RouteOrigin.BGP);
1988             LOG.info("ADD: Added Mac Fib entry rd {} prefix {} nexthop {} label {}", rd, macaddress, nextHop, l2label);
1989         } else if (addroute) {
1990             LOG.info("ADD: Adding Fib entry rd {} prefix {} nexthop {} label {} afi {}",
1991                     rd, prefix, nextHop, label, afi);
1992             // TODO: modify addFibEntryToDS signature
1993             List<String> nextHopList = Collections.singletonList(nextHop);
1994             fibDSWriter.addFibEntryToDS(rd, prefix + "/" + plen, nextHopList, encapType, label, l3vni,
1995                                         routermac, RouteOrigin.BGP);
1996             LOG.info("ADD: Added Fib entry rd {} prefix {} nexthop {} label {}", rd, prefix, nextHop, label);
1997             String vpnName = bgpUtil.getVpnNameFromRd(rd);
1998             if (vpnName != null) {
1999                 vpnLinkService.leakRouteIfNeeded(vpnName, prefix, nextHopList, label, RouteOrigin.BGP,
2000                                                  NwConstants.ADD_FLOW);
2001             }
2002         }
2003     }
2004
2005     public void onUpdateWithdrawRoute(protocol_type protocolType, String rd, String prefix, int plen, String nextHop,
2006             String macaddress) {
2007         PrefixWithdrawEvent prefixWithdrawEvent = new PrefixWithdrawEvent(protocolType,rd,prefix,plen,
2008                 nextHop,macaddress);
2009         bgpUpdatesHistory.addToHistory(TransactionType.ADD, prefixWithdrawEvent);
2010         boolean macupdate = false;
2011         if (protocolType.equals(protocol_type.PROTOCOL_EVPN)) {
2012             VpnInstanceOpDataEntry vpnInstanceOpDataEntry = bgpUtil.getVpnInstanceOpData(rd);
2013             if (vpnInstanceOpDataEntry != null) {
2014                 Uint32 vni = vpnInstanceOpDataEntry.getL3vni();
2015                 if (vpnInstanceOpDataEntry.getType() == VpnInstanceOpDataEntry.Type.L2) {
2016                     LOG.debug("Got RT2 withdraw for RD {} {} from tep {} with mac {} remote RD {}",
2017                             vpnInstanceOpDataEntry.getVpnInstanceName(), vni, nextHop, macaddress, rd);
2018                     deleteTepfromElanDS(rd, nextHop, macaddress);
2019                     LOG.debug("For rd {}. skipping fib update", rd);
2020                     macupdate = true;
2021                 }
2022             } else {
2023                 LOG.error("No corresponding vpn instance found for rd {}. Aborting.", rd);
2024                 return;
2025             }
2026         }
2027         if (macupdate) {
2028             LOG.info("Removing Mac Fib entry rd {} mac{} nexthop {} ", rd, macaddress, nextHop);
2029             fibDSWriter.removeMacEntryFromDS(rd, macaddress);
2030             LOG.info("Removed Mac Fib entry rd {} prefix {} nexthop {} ", rd, macaddress, nextHop);
2031         } else {
2032             LOG.info("REMOVE: Removing Fib entry rd {} prefix {}", rd, prefix);
2033             fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nextHop);
2034             LOG.info("REMOVE: Removed Fib entry rd {} prefix {}", rd, prefix);
2035         }
2036     }
2037
2038     //TODO: below function is for testing purpose with cli
2039     public void onUpdateWithdrawRoute(String rd, String prefix, int plen, String nexthop) {
2040         LOG.debug("Route del ** {} ** {}/{} ", rd, prefix, plen);
2041         fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix + "/" + plen, nexthop);
2042         String vpnName = bgpUtil.getVpnNameFromRd(rd);
2043         if (vpnName != null) {
2044             vpnLinkService.leakRouteIfNeeded(vpnName, prefix, null /*nextHopList*/, Uint32.ZERO /*INVALID_LABEL*/,
2045                                              RouteOrigin.BGP, NwConstants.DEL_FLOW);
2046         }
2047     }
2048
2049     public void peerDown(String ipAddress, long asNumber) {
2050         PeerDownEvent peerDownEvent = new PeerDownEvent(ipAddress,asNumber);
2051         bgpUpdatesHistory.addToHistory(TransactionType.ADD, peerDownEvent);
2052         List<String> tepIpList = bgpUtil.getDcgwTepConfig(ipAddress);
2053         if (tepIpList == null) {
2054             LOG.error("No Tep IP configured for DCGW {} on a peerDown", ipAddress);
2055             return;
2056         }
2057         tepIpList.forEach(tepIp -> {
2058             bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW);
2059         });
2060     }
2061
2062     public void peerUp(String ipAddress, long asNumber) {
2063         PeerUpEvent peerUpEvent = new PeerUpEvent(ipAddress,asNumber);
2064         bgpUpdatesHistory.addToHistory(TransactionType.ADD, peerUpEvent);
2065         List<String> tepIpList = bgpUtil.getDcgwTepConfig(ipAddress);
2066         if (tepIpList == null) {
2067             LOG.error("No Tep IP configured for DCGW {} on a peerUp", ipAddress);
2068             return;
2069         }
2070         tepIpList.forEach(tepIp -> {
2071             bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW);
2072         });
2073     }
2074
2075     private static boolean isRouteModified(Uint32 label, Uint32 labelInStaleMap) {
2076         return labelInStaleMap != null && !labelInStaleMap.equals(label);
2077     }
2078
2079     static class ReplayNbr {
2080         Neighbors nbr;
2081         boolean shouldRetry = false;
2082
2083         public Neighbors getNbr() {
2084             return nbr;
2085         }
2086
2087         public boolean isShouldRetry() {
2088             return shouldRetry;
2089         }
2090
2091         public void setShouldRetry(boolean retryNbr) {
2092             this.shouldRetry = retryNbr;
2093         }
2094
2095         ReplayNbr(Neighbors nbr, boolean shouldRetry) {
2096             this.nbr = nbr;
2097             this.shouldRetry = shouldRetry;
2098         }
2099     }
2100
2101     private static boolean replayNbrConfig(List<Neighbors> neighbors, BgpRouter br) {
2102         if (neighbors == null || neighbors.isEmpty()) {
2103             LOG.error("Replaying nbr configuration, received NULL list ");
2104             return true;
2105         }
2106
2107         List<ReplayNbr> replayNbrList = new ArrayList<>();
2108         for (Neighbors nbr : neighbors) {
2109             if (nbr != null) {
2110                 replayNbrList.add(new ReplayNbr(nbr, true));
2111             }
2112         }
2113         final int numberOfNbrRetries = 3;
2114         RetryOnException nbrRetry = new RetryOnException(numberOfNbrRetries);
2115         do {
2116             for (ReplayNbr replayNbr : replayNbrList) {
2117                 if (!replayNbr.isShouldRetry()) {
2118                     continue;
2119                 }
2120                 boolean replayDone = false;
2121                 LOG.debug("Replaying addNbr {}", replayNbr.getNbr().getAddress().getValue());
2122                 replayDone = false;
2123                 try {
2124                     final String md5password = extractMd5Secret(replayNbr.getNbr());
2125                     br.addNeighbor(replayNbr.getNbr().getAddress().getValue(),
2126                             replayNbr.getNbr().getRemoteAs().longValue(), md5password);
2127                     replayDone = true;
2128                 } catch (TApplicationException tae) {
2129                     LOG.debug("Replaying addNbr {}, tapplicationexception: ",
2130                             replayNbr.getNbr().getAddress().getValue(), tae);
2131                     if (tae.getType() == BgpRouterException.BGP_ERR_PEER_EXISTS) {
2132                         LOG.debug("Replaying addNbr Neighbor already present");
2133                         replayDone = true;
2134                     } else {
2135                         LOG.error("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), tae);
2136                     }
2137                 } catch (TException | BgpRouterException eNbr) {
2138                     LOG.debug("Replaying addNbr {}, exception: ", replayNbr.getNbr().getAddress().getValue(), eNbr);
2139                 }
2140
2141                 LOG.debug("Replay addNbr {} successful", replayNbr.getNbr().getAddress().getValue());
2142
2143                 //Update Source handling
2144                 UpdateSource us = replayNbr.getNbr().getUpdateSource();
2145                 if (us != null) {
2146                     LOG.debug("Replaying updatesource {} to peer {}", us.getSourceIp().getValue(),
2147                             us.getPeerIp().getValue());
2148                     try {
2149                         br.addUpdateSource(us.getPeerIp().getValue(),
2150                                 us.getSourceIp().getValue());
2151                     } catch (TException | BgpRouterException eUs) {
2152                         LOG.debug("Replaying UpdateSource for Nbr {}, exception:",
2153                                 replayNbr.getNbr().getAddress().getValue(), eUs);
2154                     }
2155                     LOG.debug("Replay updatesource {} successful", us.getSourceIp().getValue());
2156                 }
2157                 //Ebgp Multihope
2158                 EbgpMultihop en = replayNbr.getNbr().getEbgpMultihop();
2159                 if (en != null) {
2160                     try {
2161                         br.addEbgpMultihop(en.getPeerIp().getValue(),
2162                                 en.getNhops().intValue());
2163                     } catch (TException | BgpRouterException eEbgpMhop) {
2164                         LOG.debug("Replaying EbgpMultihop for Nbr {}, exception: ",
2165                                 replayNbr.getNbr().getAddress().getValue(), eEbgpMhop);
2166                     }
2167                 }
2168
2169                 //keyAddressFamiliesMap
2170                 Map<AddressFamiliesKey, AddressFamilies> keyAddressFamiliesMap
2171                         = replayNbr.getNbr().getAddressFamilies();
2172                 if (keyAddressFamiliesMap != null) {
2173                     for (AddressFamilies af : keyAddressFamiliesMap.values()) {
2174                         af_afi afi = af_afi.findByValue(af.getAfi().intValue());
2175                         af_safi safi = af_safi.findByValue(af.getSafi().intValue());
2176                         try {
2177                             br.addAddressFamily(af.getPeerIp().getValue(), afi, safi);
2178                         } catch (TException | BgpRouterException eAFs) {
2179                             LOG.debug("Replaying AddressFamily for Nbr {}, exception:",
2180                                     replayNbr.getNbr().getAddress().getValue(), eAFs);
2181                         }
2182                     }
2183                 }
2184                 //replay is success --> no need to replay this nbr in next iteration.
2185                 replayNbr.setShouldRetry(replayDone ? false : true);
2186             }
2187         } while (nbrRetry.decrementAndRetry());
2188         boolean replaySuccess = true;
2189         for (ReplayNbr replayNbr : replayNbrList) {
2190             replaySuccess = replaySuccess && !replayNbr.isShouldRetry();
2191             if (replaySuccess == false) {
2192                 LOG.error("replayNbrConfig: will be cancelling stale cleanup, cfg nbr: {} Failed:",
2193                         replayNbr.getNbr().getAddress().getValue());
2194             }
2195         }
2196         return replaySuccess;
2197     }
2198
2199     public String getConfigHost() {
2200         if (config == null) {
2201             return hostStartup;
2202         }
2203         ConfigServer ts = config.getConfigServer();
2204         return ts == null ? hostStartup : ts.getHost().getValue();
2205     }
2206
2207     public int getConfigPort() {
2208         if (config == null) {
2209             return Integer.parseInt(portStartup);
2210         }
2211         ConfigServer ts = config.getConfigServer();
2212         return ts == null ? Integer.parseInt(portStartup) :
2213                 ts.getPort().intValue();
2214     }
2215
2216     public Bgp getConfig() {
2217         AtomicInteger bgpDSretryCount = new AtomicInteger(DS_RETRY_COUNT);
2218         while (0 != bgpDSretryCount.decrementAndGet()) {
2219             try {
2220                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
2221                         InstanceIdentifier.create(Bgp.class)).orElse(null);
2222             } catch (InterruptedException | ExecutionException e) {
2223                 //Config DS may not be up, so sleep for 1 second and retry
2224                 LOG.debug("failed to get bgp config, may be DS is yet in consistent state(?)", e);
2225                 try {
2226                     Thread.sleep(WAIT_TIME_BETWEEN_EACH_TRY_MILLIS);
2227                 } catch (InterruptedException timerEx) {
2228                     LOG.debug("WAIT_TIME_BETWEEN_EACH_TRY_MILLIS, Timer got interrupted while waiting for"
2229                             + "config DS availability", timerEx);
2230                 }
2231             }
2232         }
2233         LOG.error("failed to get bgp config");
2234         return null;
2235     }
2236
2237     @SuppressWarnings("checkstyle:IllegalCatch")
2238     public synchronized boolean replay() throws InterruptedException, TimeoutException, ExecutionException {
2239         boolean replaySucceded = true;
2240         boolean doRouteSync = false;
2241         String host = getConfigHost();
2242         int port = getConfigPort();
2243         LOG.error("connecting  to bgp host {} ", host);
2244         boolean res = bgpRouter.connect(host, port);
2245         if (!res) {
2246             LOG.error("Cannot connect to BGP config server at {} {}", host, port);
2247             return replaySucceded;
2248         }
2249         config = getConfig();
2250         if (config == null) {
2251             LOG.error("bgp config is empty nothing to push to bgp");
2252             return replaySucceded;
2253         }
2254         BgpRouter br = bgpRouter;
2255         AsId asId = config.getAsId();
2256         if (asId == null) {
2257             LOG.error("bgp as-id is null");
2258             return replaySucceded;
2259         }
2260         long asNum = asId.getLocalAs().toJava();
2261         IpAddress routerId = asId.getRouterId();
2262         String rid = routerId == null ? "" : routerId.stringValue();
2263         int stalepathTime = (int) getStalePathtime(bgpGrRestartTime, config.getAsId());
2264         boolean announceFbit = true;
2265         boolean replayDone = false;
2266         final int numberOfStartBgpRetries = 3;
2267         RetryOnException startBgpRetry = new RetryOnException(numberOfStartBgpRetries);
2268         do {
2269             try {
2270                 LOG.debug("Replaying BGPConfig ");
2271                 br.startBgp(asNum, rid, bgpKaTime, bgpHoldTime, stalepathTime, announceFbit);
2272                 LOG.debug("Replay BGPConfig successful");
2273                 replayDone = true;
2274                 break;
2275             } catch (BgpRouterException bre) {
2276                 if (bre.getErrorCode() == BgpRouterException.BGP_ERR_ACTIVE) {
2277                     LOG.debug("Starting the routesync for exception", bre);
2278                     startBgpRetry.errorOccured();
2279                     if (!startBgpRetry.shouldRetry()) {
2280                         doRouteSync = true;
2281                         replayDone = true;
2282                     }
2283                 } else {
2284                     LOG.error("Replay: startBgp() received exception error {} : ",
2285                             bre.getErrorCode(), bre);
2286                     startBgpRetry.errorOccured();
2287                 }
2288             } catch (TApplicationException tae) {
2289                 if (tae.getType() == BgpRouterException.BGP_ERR_ACTIVE) {
2290                     LOG.debug("Starting the routesync for exception", tae);
2291                     startBgpRetry.errorOccured();
2292                     if (!startBgpRetry.shouldRetry()) {
2293                         doRouteSync = true;
2294                         replayDone = true;
2295                     }
2296                 } else if (tae.getType() == BgpRouterException.BGP_ERR_COMMON_FAILURE) {
2297                     LOG.debug("Starting the routesync for AS-ID started exception", tae);
2298                     startBgpRetry.errorOccured();
2299                     if (!startBgpRetry.shouldRetry()) {
2300                         doRouteSync = true;
2301                         replayDone = true;
2302                     }
2303                 } else {
2304                     LOG.error("Replay: startBgp() received exception type {}: ",
2305                             tae.getType(), tae);
2306                     startBgpRetry.errorOccured();
2307                 }
2308             } catch (Exception e) {
2309                 //not unusual. We may have restarted & BGP is already on
2310                 LOG.error("Replay:startBgp() received exception: ", e);
2311                 startBgpRetry.errorOccured();
2312             }
2313         } while (startBgpRetry.shouldRetry());
2314
2315         replaySucceded = replayDone;
2316
2317         startBgpCountersTask();
2318         startBgpAlarmsTask();
2319
2320         /*
2321          * commenting this due to a bug with QBGP. Will uncomment once QBGP fix is done.
2322          * This wont have any functional impacts
2323          */
2324         //try {
2325         //    br.delayEOR(delayEorSeconds);
2326         //} catch (TException | BgpRouterException e) {
2327         //    LOG.error("Replay: delayEOR() number of seconds to wait for EOR from ODL:", e);
2328         //}
2329
2330         BfdConfig bfdConfig = bgpUtil.getBfdConfig();
2331         if (bfdConfig != null) {
2332             if (bfdConfig.isBfdEnabled()) {
2333                 LOG.debug("Replaying bfd config min-rx {} min-tx {} detect-mul {} mhop {}",
2334                         bfdConfig.getMinRx(), bfdConfig.getMinTx(),
2335                         bfdConfig.getDetectMult(), bfdConfig.isMultihop());
2336                 try {
2337                     br.addBfd(bfdConfig.getDetectMult().intValue(), bfdConfig.getMinRx().intValue(),
2338                             bfdConfig.getMinTx().intValue(), bfdConfig.isMultihop());
2339                 } catch (TApplicationException tae) {
2340                     if (tae.getType() == BgpRouterException.BGP_ERR_PEER_EXISTS) {
2341                         LOG.debug("Replay:addBfd() received exception", tae);
2342                     } else {
2343                         LOG.error("Replay:addBfd() received exception", tae);
2344                     }
2345                 } catch (TException | BgpRouterException e) {
2346                     LOG.error("Replay:addBfd() received exception", e);
2347                 }
2348             }
2349         }
2350
2351         Map<NeighborsKey, Neighbors> keyNeighborsMap = config.getNeighborsContainer() == null ? null
2352                 : config.getNeighborsContainer().getNeighbors();
2353         if (keyNeighborsMap != null) {
2354             LOG.error("configuring existing Neighbors present for replay total keyNeighborsMap {}",
2355                     keyNeighborsMap.values().size());
2356             boolean neighborConfigReplayResult
2357                     = replayNbrConfig(new ArrayList<Neighbors>(keyNeighborsMap.values()), br);
2358             if (neighborConfigReplayResult == false) {
2359                 replaySucceded = false;
2360             }
2361         } else {
2362             LOG.error("no Neighbors present for replay config ");
2363         }
2364
2365         Logging logging = config.getLogging();
2366         if (logging != null) {
2367             try {
2368                 br.setLogging(logging.getFile(), logging.getLevel());
2369             } catch (TException | BgpRouterException e) {
2370                 LOG.error("Replay:setLogging() received exception", e);
2371             }
2372         }
2373
2374         GracefulRestart gracefulRestart = config.getGracefulRestart();
2375         bgpGrRestartTime = ((gracefulRestart != null)
2376                 ? gracefulRestart.getStalepathTime().intValue() : bgpGrRestartTime);
2377         try {
2378             br.addGracefulRestart(bgpGrRestartTime);
2379         } catch (Exception e) {
2380             LOG.error("Replay:addGr() received exception: ", e);
2381         }
2382         Map<VrfsKey, Vrfs> keyVrfsMap = config.getVrfsContainer() == null ? null
2383                 : config.getVrfsContainer().getVrfs();
2384         if (keyVrfsMap == null) {
2385             keyVrfsMap = new HashMap<VrfsKey, Vrfs>();
2386         }
2387         for (Vrfs vrf : keyVrfsMap.values()) {
2388             for (AddressFamiliesVrf adf : vrf.getAddressFamiliesVrf().values()) {
2389                 try {
2390                     br.addVrf(BgpUtil.getLayerType(adf), vrf.getRd(), vrf.getImportRts(),
2391                             vrf.getExportRts(), adf.getAfi().toJava(), adf.getSafi().toJava());
2392                 } catch (TException | BgpRouterException e) {
2393                     LOG.error("Replay:addVrf() received exception", e);
2394                 }
2395             }
2396         }
2397
2398
2399         Map<NetworksKey, Networks> keyNetworksMap = config.getNetworksContainer() == null ? null
2400                 : config.getNetworksContainer().getNetworks();
2401         if (keyNetworksMap != null) {
2402             for (Networks net : keyNetworksMap.values()) {
2403                 String rd = net.getRd();
2404                 String pfxlen = net.getPrefixLen();
2405                 String nh = net.getNexthop().getValue();
2406                 Long label = net.getLabel().toJava();
2407                 int lbl = label == null ? 0 : label.intValue();
2408                 int l3vni = net.getL3vni() == null ? 0 : net.getL3vni().intValue();
2409                 int l2vni = net.getL2vni() == null ? 0 : net.getL2vni().intValue();
2410                 if (rd == null && lbl > 0) {
2411                     //LU prefix is being deleted.
2412                     rd = Integer.toString(lbl);
2413                 }
2414
2415                 BgpControlPlaneType protocolType = net.getBgpControlPlaneType();
2416                 int ethernetTag = net.getEthtag().intValue();
2417                 String esi = net.getEsi();
2418                 String macaddress = net.getMacaddress();
2419                 EncapType encapType = net.getEncapType();
2420                 String routerMac = net.getRoutermac();
2421
2422                 try {
2423                     br.addPrefix(rd, pfxlen, nh, lbl, l3vni, l2vni,
2424                             BgpUtil.convertToThriftProtocolType(protocolType),
2425                             ethernetTag, esi, macaddress, BgpUtil.convertToThriftEncapType(encapType), routerMac);
2426                 } catch (TException | BgpRouterException e) {
2427                     LOG.error("Replay:addPfx() received exception", e);
2428                 }
2429             }
2430         }
2431
2432
2433         Map<MultipathKey, Multipath> keyMultipathMap = config.getMultipathContainer() == null ? null
2434                 : config.getMultipathContainer().getMultipath();
2435
2436         if (keyMultipathMap != null) {
2437             for (Multipath multipath : keyMultipathMap.values()) {
2438                 if (multipath != null) {
2439                     af_afi afi = af_afi.findByValue(multipath.getAfi().intValue());
2440                     af_safi safi = af_safi.findByValue(multipath.getSafi().intValue());
2441
2442                     try {
2443                         if (multipath.isMultipathEnabled()) {
2444                             br.enableMultipath(afi, safi);
2445                         } else {
2446                             br.disableMultipath(afi, safi);
2447                         }
2448                     } catch (TException | BgpRouterException e) {
2449                         LOG.info("Replay:keyMultipathMap() received exception", e);
2450                     }
2451                 }
2452             }
2453         }
2454         Map<VrfMaxpathKey, VrfMaxpath> keyVrfMaxpathMap = config.getVrfMaxpathContainer() == null ? null
2455                 : config.getVrfMaxpathContainer().getVrfMaxpath();
2456         if (keyVrfMaxpathMap != null) {
2457             for (VrfMaxpath vrfMaxpath : keyVrfMaxpathMap.values()) {
2458                 try {
2459                     br.multipaths(vrfMaxpath.getRd(), vrfMaxpath.getMaxpaths().toJava());
2460                 } catch (TException | BgpRouterException e) {
2461                     LOG.info("Replay:vrfMaxPath() received exception", e);
2462                 }
2463             }
2464         }
2465
2466         //send End of Rib Marker to Qthriftd.
2467         final int numberOfEORRetries = 3;
2468         RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
2469         do {
2470             try {
2471                 br.sendEOR();
2472                 LOG.debug("Replay sendEOR() successful");
2473                 break;
2474             } catch (Exception e) {
2475                 eorRetry.errorOccured();
2476                 LOG.error("Replay:sedEOR() received exception:", e);
2477             }
2478         } while (eorRetry.shouldRetry());
2479
2480         if (doRouteSync) {
2481             LOG.debug("starting route sync for Thrift BGP_ERR_COMMON_FAILURE exception "
2482                       + "happened earlier");
2483             doRouteSync();
2484         }
2485
2486         return replaySucceded;
2487     }
2488
2489     private <T extends DataObject> void update(InstanceIdentifier<T> iid, T dto) {
2490         bgpUtil.update(iid, dto);
2491     }
2492
2493     private <T extends DataObject> void delete(InstanceIdentifier<T> iid) {
2494         bgpUtil.delete(iid);
2495     }
2496
2497     public void startConfig(String bgpHost, int thriftPort) {
2498         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2499                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2500         InstanceIdentifier<ConfigServer> iid = iib.build();
2501         Ipv4Address ipAddr = new Ipv4Address(bgpHost);
2502         ConfigServer dto = new ConfigServerBuilder().setHost(ipAddr)
2503                 .setPort((long) thriftPort).build();
2504         update(iid, dto);
2505     }
2506
2507     public void startBgp(long as, String routerId, int spt, boolean fbit) {
2508         IpAddress rid = routerId == null ? null : IpAddressBuilder.getDefaultInstance(routerId);
2509         Long staleTime = (long) spt;
2510         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2511                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2512         InstanceIdentifier<AsId> iid = iib.build();
2513         AsId dto = new AsIdBuilder().setLocalAs(as)
2514                 .setRouterId(rid)
2515                 .setStalepathTime(staleTime)
2516                 .setAnnounceFbit(fbit).build();
2517         update(iid, dto);
2518     }
2519
2520     public void startBfd(long detectMult, long minRx, long minTx, boolean multiHop) {
2521         InstanceIdentifier.InstanceIdentifierBuilder<BfdConfig> iib =
2522                 InstanceIdentifier.builder(BfdConfig.class);
2523         InstanceIdentifier<BfdConfig> iid = iib.build();
2524         BfdConfig dto = new BfdConfigBuilder()
2525                 .setBfdEnabled(true)
2526                 .setMultihop(multiHop)
2527                 .setMinRx(minRx)
2528                 .setMinTx(minTx)
2529                 .setDetectMult(detectMult)
2530                 .build();
2531         update(iid, dto);
2532     }
2533
2534     public void addDcgwTep(String dcgwIp, String tepIp) {
2535         InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2536                 InstanceIdentifier.builder(Bgp.class)
2537                         .child(DcgwTepList.class)
2538                         .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2539         InstanceIdentifier<DcgwTep> iid = iib.build();
2540         ArrayList<String> tepList = new ArrayList<String>();
2541         tepList.add(tepIp);
2542         DcgwTep dto = new DcgwTepBuilder().setDcGwIp(dcgwIp).setTepIps(tepList)
2543                 .build();
2544         update(iid, dto);
2545         bgpUtil.removeOrUpdateLBGroups(tepIp,NwConstants.MOD_FLOW);
2546     }
2547
2548     public void addLogging(String fileName, String logLevel) {
2549         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2550                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2551         InstanceIdentifier<Logging> iid = iib.build();
2552         Logging dto = new LoggingBuilder().setFile(fileName)
2553                 .setLevel(logLevel).build();
2554         update(iid, dto);
2555     }
2556
2557     public void addGracefulRestart(int staleTime) {
2558         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2559                 InstanceIdentifier.builder(Bgp.class).child(GracefulRestart.class);
2560         InstanceIdentifier<GracefulRestart> iid = iib.build();
2561         GracefulRestart dto = new GracefulRestartBuilder()
2562                 .setStalepathTime((long) staleTime).build();
2563         update(iid, dto);
2564     }
2565
2566     public void addNeighbor(
2567             String nbrIp, long remoteAs, @Nullable final TcpMd5SignaturePasswordType md5Secret) {
2568         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2569         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2570                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2571                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2572         InstanceIdentifier<Neighbors> iid = iib.build();
2573         TcpSecurityOption tcpSecOption = null;
2574         if (md5Secret != null) {
2575             tcpSecOption = new TcpMd5SignatureOptionBuilder().setTcpMd5SignaturePassword(md5Secret).build();
2576         } // else let tcpSecOption be null
2577         Neighbors dto = new NeighborsBuilder().setAddress(nbrAddr)
2578                 .setRemoteAs(remoteAs).setTcpSecurityOption(tcpSecOption).build();
2579         update(iid, dto);
2580     } // public addNeighbor(nbrIp, remoteAs, md5Secret)
2581
2582     public void addUpdateSource(String nbrIp, String srcIp) {
2583         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2584         Ipv4Address srcAddr = new Ipv4Address(srcIp);
2585         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2586                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2587                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2588                         .child(UpdateSource.class);
2589         InstanceIdentifier<UpdateSource> iid = iib.build();
2590         UpdateSource dto = new UpdateSourceBuilder().setPeerIp(nbrAddr)
2591                 .setSourceIp(srcAddr).build();
2592         update(iid, dto);
2593     }
2594
2595     public void addEbgpMultihop(String nbrIp, int hops) {
2596         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2597         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2598                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2599                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2600                         .child(EbgpMultihop.class);
2601         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2602         EbgpMultihop dto = new EbgpMultihopBuilder().setPeerIp(nbrAddr)
2603                 .setNhops((long) hops).build();
2604         update(iid, dto);
2605     }
2606
2607     public void addAddressFamily(String nbrIp, int afi, int safi) {
2608         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2609         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2610                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2611                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2612                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2613         InstanceIdentifier<AddressFamilies> iid = iib.build();
2614         AddressFamilies dto = new AddressFamiliesBuilder().setPeerIp(nbrAddr)
2615                 .setAfi((long) afi).setSafi((long) safi).build();
2616         update(iid, dto);
2617     }
2618
2619     public void addPrefix(String rd, String macAddress, String pfx, List<String> nhList,
2620               VrfEntry.EncapType encapType, Uint32 lbl, Uint32 l3vni, Uint32 l2vni, String gatewayMac) {
2621         for (String nh : nhList) {
2622             Ipv4Address nexthop = nh != null ? new Ipv4Address(nh) : null;
2623             Uint32 label = lbl;
2624             InstanceIdentifier<Networks> iid = InstanceIdentifier.builder(Bgp.class)
2625                     .child(NetworksContainer.class)
2626                     .child(Networks.class, new NetworksKey(pfx, rd)).build();
2627             NetworksBuilder networksBuilder = new NetworksBuilder().setRd(rd).setPrefixLen(pfx).setNexthop(nexthop)
2628                                                 .setLabel(label).setEthtag(BgpConstants.DEFAULT_ETH_TAG);
2629             buildVpnEncapSpecificInfo(networksBuilder, encapType, label, l3vni, l2vni, macAddress, gatewayMac);
2630             update(iid, networksBuilder.build());
2631         }
2632     }
2633
2634     private static void buildVpnEncapSpecificInfo(NetworksBuilder builder, VrfEntry.EncapType encapType, Uint32 label,
2635                                                   Uint32 l3vni, Uint32 l2vni, String macAddress, String gatewayMac) {
2636         if (encapType.equals(VrfEntry.EncapType.Mplsgre)) {
2637             builder.setLabel(label).setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLL3VPN)
2638                     .setEncapType(EncapType.GRE);
2639         } else {
2640             builder.setL3vni(l3vni).setL2vni(l2vni).setMacaddress(macAddress).setRoutermac(gatewayMac)
2641                     .setBgpControlPlaneType(BgpControlPlaneType.PROTOCOLEVPN).setEncapType(EncapType.VXLAN);
2642         }
2643     }
2644
2645     // TODO: add LayerType as arg - supports command
2646     public void addVrf(String rd, List<String> irts, List<String> erts, AddressFamily addressFamily) {
2647         Vrfs vrf = bgpUtil.getVrfFromRd(rd);
2648         List<AddressFamiliesVrf> adfList = new ArrayList<>(1);
2649         if (vrf != null) {
2650             adfList = new ArrayList<AddressFamiliesVrf>(vrf.getAddressFamiliesVrf().values());
2651         }
2652         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2653         if (addressFamily.equals(AddressFamily.IPV4)) {
2654             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2655             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2656         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2657             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2658             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2659         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2660             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2661             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2662         }
2663         AddressFamiliesVrf adf = adfBuilder.build();
2664         adfList.add(adf);
2665         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib = InstanceIdentifier.builder(Bgp.class)
2666                 .child(VrfsContainer.class)
2667                 .child(Vrfs.class, new VrfsKey(rd));
2668         InstanceIdentifier<Vrfs> iid = iib.build();
2669         Vrfs dto = new VrfsBuilder().setRd(rd).setImportRts(irts)
2670             .setExportRts(erts).setAddressFamiliesVrf(adfList).build();
2671
2672         List<AddressFamiliesVrf> listAdFamilies = mapNewAdFamily.get(rd);
2673         if (listAdFamilies != null) {
2674             listAdFamilies.add(adf);
2675         } else {
2676             mapNewAdFamily.put(rd, adfList);
2677         }
2678
2679         try {
2680             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, iid, dto);
2681         } catch (TransactionCommitFailedException e) {
2682             LOG.error("Error adding VRF to datastore", e);
2683             throw new RuntimeException(e);
2684         }
2685
2686         // enable multipath by default in all VRFs
2687         setMultipaths(rd, BgpConstants.BGP_DEFAULT_MULTIPATH);
2688     }
2689
2690     public void stopConfig() {
2691         InstanceIdentifier.InstanceIdentifierBuilder<ConfigServer> iib =
2692                 InstanceIdentifier.builder(Bgp.class).child(ConfigServer.class);
2693         InstanceIdentifier<ConfigServer> iid = iib.build();
2694         delete(iid);
2695     }
2696
2697     public void stopBgp() {
2698         InstanceIdentifier.InstanceIdentifierBuilder<AsId> iib =
2699                 InstanceIdentifier.builder(Bgp.class).child(AsId.class);
2700         InstanceIdentifier<AsId> iid = iib.build();
2701         delete(iid);
2702     }
2703
2704     public void stopBfd() {
2705         InstanceIdentifier.InstanceIdentifierBuilder<BfdConfig> iib =
2706                 InstanceIdentifier.builder(BfdConfig.class);
2707         InstanceIdentifier<BfdConfig> iid = iib.build();
2708         delete(iid);
2709     }
2710
2711     public void delDcgwTep(String dcgwIp, String tepIp) {
2712         if (tepIp == null) {
2713             InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2714                     InstanceIdentifier.builder(Bgp.class)
2715                             .child(DcgwTepList.class)
2716                             .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2717             InstanceIdentifier<DcgwTep> iid = iib.build();
2718             delete(iid);
2719         } else {
2720             InstanceIdentifier.InstanceIdentifierBuilder<DcgwTep> iib =
2721                     InstanceIdentifier.builder(Bgp.class)
2722                             .child(DcgwTepList.class)
2723                             .child(DcgwTep.class, new DcgwTepKey(dcgwIp));
2724             InstanceIdentifier<DcgwTep> iid = iib.build();
2725             List<String> tepIpList = bgpUtil.getDcgwTepConfig(dcgwIp);
2726             if (tepIpList == null) {
2727                 LOG.error("No Tep IP configured for DCGW {} on deleting the dcgwtep", dcgwIp);
2728                 return;
2729             }
2730             List<String> newTepIpList = new ArrayList<String>();
2731             tepIpList.forEach(tep -> {
2732                 if (!tep.equals(tepIp)) {
2733                     newTepIpList.add(tep);
2734                 }
2735             });
2736             DcgwTep dto = new DcgwTepBuilder().setDcGwIp(dcgwIp).setTepIps(newTepIpList)
2737                     .build();
2738             try {
2739                 SingleTransactionDataBroker.syncWrite(dataBroker,
2740                         LogicalDatastoreType.CONFIGURATION, iid, dto);
2741             } catch (TransactionCommitFailedException e) {
2742                 LOG.error("delDcgwTep: Error deleting DCGW Tep", e);
2743                 throw new RuntimeException(e);
2744             }
2745         }
2746     }
2747
2748     public void delLogging() {
2749         InstanceIdentifier.InstanceIdentifierBuilder<Logging> iib =
2750                 InstanceIdentifier.builder(Bgp.class).child(Logging.class);
2751         InstanceIdentifier<Logging> iid = iib.build();
2752         delete(iid);
2753     }
2754
2755     public void delGracefulRestart() {
2756         InstanceIdentifier.InstanceIdentifierBuilder<GracefulRestart> iib =
2757                 InstanceIdentifier.builder(Bgp.class)
2758                         .child(GracefulRestart.class);
2759         InstanceIdentifier<GracefulRestart> iid = iib.build();
2760         delete(iid);
2761     }
2762
2763     public void delNeighbor(String nbrIp) {
2764         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2765         InstanceIdentifier.InstanceIdentifierBuilder<Neighbors> iib =
2766                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2767                         .child(Neighbors.class, new NeighborsKey(nbrAddr));
2768         InstanceIdentifier<Neighbors> iid = iib.build();
2769         delete(iid);
2770     }
2771
2772     public void delUpdateSource(String nbrIp) {
2773         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2774         InstanceIdentifier.InstanceIdentifierBuilder<UpdateSource> iib =
2775                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2776                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2777                         .child(UpdateSource.class);
2778         InstanceIdentifier<UpdateSource> iid = iib.build();
2779         delete(iid);
2780     }
2781
2782     public void delEbgpMultihop(String nbrIp) {
2783         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2784         InstanceIdentifier.InstanceIdentifierBuilder<EbgpMultihop> iib =
2785                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2786                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2787                         .child(EbgpMultihop.class);
2788         InstanceIdentifier<EbgpMultihop> iid = iib.build();
2789         delete(iid);
2790     }
2791
2792     public void delAddressFamily(String nbrIp, int afi, int safi) {
2793         Ipv4Address nbrAddr = new Ipv4Address(nbrIp);
2794         InstanceIdentifier.InstanceIdentifierBuilder<AddressFamilies> iib =
2795                 InstanceIdentifier.builder(Bgp.class).child(NeighborsContainer.class)
2796                         .child(Neighbors.class, new NeighborsKey(nbrAddr))
2797                         .child(AddressFamilies.class, new AddressFamiliesKey((long) afi, (long) safi));
2798         InstanceIdentifier<AddressFamilies> iid = iib.build();
2799         delete(iid);
2800     }
2801
2802     public void delPrefix(String rd, String pfx) {
2803         InstanceIdentifier.InstanceIdentifierBuilder<Networks> iib =
2804                 InstanceIdentifier.builder(Bgp.class).child(NetworksContainer.class)
2805                         .child(Networks.class, new NetworksKey(pfx, rd));
2806         InstanceIdentifier<Networks> iid = iib.build();
2807         delete(iid);
2808     }
2809
2810     public boolean delVrf(String rd, AddressFamily addressFamily) {
2811         if (addressFamily == null) {
2812             LOG.error("delVrf: vrf {}, addressFamily invalid", rd);
2813             return false;
2814         }
2815         delMultipaths(rd);
2816         AddressFamiliesVrfBuilder adfBuilder = new AddressFamiliesVrfBuilder();
2817         if (addressFamily.equals(AddressFamily.IPV4)) {
2818             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2819             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2820         } else if (addressFamily.equals(AddressFamily.IPV6)) {
2821             adfBuilder.setAfi((long) af_afi.AFI_IPV6.getValue());
2822             adfBuilder.setSafi((long) af_safi.SAFI_MPLS_VPN.getValue());
2823         } else if (addressFamily.equals(AddressFamily.L2VPN)) {
2824             adfBuilder.setAfi((long) af_afi.AFI_IP.getValue());
2825             adfBuilder.setSafi((long) af_safi.SAFI_EVPN.getValue());
2826         }
2827         LOG.debug("delVrf: Received Delete VRF : rd:{}, address family: {} {}", rd,
2828                 adfBuilder.getAfi(), adfBuilder.getSafi());
2829
2830         Vrfs vrfOriginal = bgpUtil.getVrfFromRd(rd);
2831         if (vrfOriginal == null) {
2832             LOG.error("delVrf: no vrf with existing rd {}. step aborted", rd);
2833             return false;
2834         }
2835
2836         InstanceIdentifier.InstanceIdentifierBuilder<Vrfs> iib =
2837                 InstanceIdentifier.builder(Bgp.class).child(VrfsContainer.class)
2838                         .child(Vrfs.class, new VrfsKey(rd));
2839
2840         InstanceIdentifier<Vrfs> iid = iib.build();
2841
2842         @SuppressWarnings("static-access")
2843         InstanceIdentifier<Bgp> iid6 =  iid.builder(Bgp.class).build()
2844                 .child(MultipathContainer.class)
2845                 .child(Multipath.class, new MultipathKey(adfBuilder.getAfi(), adfBuilder.getSafi())).create(Bgp.class);
2846         InstanceIdentifierBuilder<Vrfs> iib3 =
2847                 iid6.child(VrfsContainer.class).child(Vrfs.class, new VrfsKey(rd)).builder();
2848         InstanceIdentifier<Vrfs> iidFinal = iib3.build();
2849
2850         //** update or delete the vrfs with the rest of AddressFamilies already present in the last list
2851         AddressFamiliesVrf adfToDel = adfBuilder.build();
2852         List<AddressFamiliesVrf> adfListOriginal = new ArrayList<>(vrfOriginal.nonnullAddressFamiliesVrf().values());
2853         List<AddressFamiliesVrf> adfListToRemoveFromOriginal = new ArrayList<>();
2854         adfListOriginal.forEach(adf -> {
2855             if (adf.equals(adfToDel)) {
2856                 adfListToRemoveFromOriginal.add(adfToDel);
2857                 return;
2858             }
2859         });
2860         for (AddressFamiliesVrf adfToRemove : adfListToRemoveFromOriginal) {
2861             adfListOriginal.remove(adfToRemove);
2862             try {
2863                 SingleTransactionDataBroker.syncWrite(dataBroker,
2864                         LogicalDatastoreType.CONFIGURATION, iid, vrfOriginal);
2865             } catch (TransactionCommitFailedException e) {
2866                 LOG.error("delVrf: Error updating VRF to datastore", e);
2867                 throw new RuntimeException(e);
2868             }
2869         }
2870         if (adfListOriginal.isEmpty()) {
2871             LOG.debug("delVrf: delete iid: {}", iidFinal);
2872             delete(iidFinal);
2873             return true;
2874         }
2875         // not all is removed
2876         return false;
2877     }
2878
2879     public void setMultipathStatus(af_afi afi, af_safi safi, boolean enable) {
2880         long lafi = afi.getValue();
2881         long lsafi = safi.getValue();
2882
2883         InstanceIdentifier.InstanceIdentifierBuilder<Multipath> iib =
2884                 InstanceIdentifier
2885                         .builder(Bgp.class).child(MultipathContainer.class)
2886                         .child(Multipath.class,
2887                                 new MultipathKey(Long.valueOf(afi.getValue()), Long.valueOf(safi.getValue())));
2888
2889         Multipath dto = new MultipathBuilder().setAfi(lafi).setSafi(lsafi).setMultipathEnabled(enable).build();
2890         update(iib.build(), dto);
2891     }
2892
2893     public void setMultipaths(String rd, int maxpath) {
2894         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
2895                 InstanceIdentifier
2896                         .builder(Bgp.class).child(VrfMaxpathContainer.class)
2897                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
2898
2899         VrfMaxpath dto = new VrfMaxpathBuilder().setRd(rd).setMaxpaths(maxpath).build();
2900         update(iib.build(), dto);
2901     }
2902
2903     public void delMultipaths(String rd) {
2904         InstanceIdentifier.InstanceIdentifierBuilder<VrfMaxpath> iib =
2905                 InstanceIdentifier.builder(Bgp.class).child(VrfMaxpathContainer.class)
2906                         .child(VrfMaxpath.class, new VrfMaxpathKey(rd));
2907         InstanceIdentifier<VrfMaxpath> iid = iib.build();
2908         delete(iid);
2909     }
2910
2911     /*
2912     * Remove Stale Marked Routes after timer expiry.
2913     */
2914     private class RouteCleanup implements Callable<Integer> {
2915
2916         @Override
2917         public Integer call() {
2918             totalCleared = 0;
2919             try {
2920                 if (staledFibEntriesMap.isEmpty()) {
2921                     LOG.info("BGP: RouteCleanup timertask tirggered but STALED FIB MAP is EMPTY");
2922                 } else {
2923                     for (String rd : staledFibEntriesMap.keySet()) {
2924                         if (Thread.interrupted()) {
2925                             return 0;
2926                         }
2927                         Map<String, Uint32> map = staledFibEntriesMap.get(rd);
2928                         if (map != null) {
2929                             for (String key : map.keySet()) {
2930                                 if (Thread.interrupted()) {
2931                                     return 0;
2932                                 }
2933                                 String prefix = extractPrefix(key);
2934                                 String nextHop = extractNextHop(key);
2935                                 totalCleared++;
2936                                 LOG.debug("BGP: RouteCleanup deletePrefix called for : rd:{}, prefix{}, nextHop:{}",
2937                                         rd, prefix, nextHop);
2938                                 fibDSWriter.removeOrUpdateFibEntryFromDS(rd, prefix, nextHop);
2939                             }
2940                         }
2941                     }
2942                 }
2943             } finally {
2944                 staledFibEntriesMap.clear();
2945             }
2946             LOG.error("cleared {} stale routes after bgp restart", totalCleared);
2947             return 0;
2948         }
2949     }
2950
2951     /*
2952      * BGP restart scenario, ODL-BGP manager was/is running.
2953      * On re-sync notification, Get a copy of FIB database.
2954      */
2955     public void createStaleFibMap() {
2956         totalStaledCount = 0;
2957         try {
2958             staledFibEntriesMap.clear();
2959             fibDSWriter.clearFibMap();
2960             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2961
2962             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2963                     LogicalDatastoreType.CONFIGURATION, id);
2964             if (fibEntries.isPresent()) {
2965                 Map<VrfTablesKey, VrfTables> staleVrfTablesMap = fibEntries.get().getVrfTables();
2966                 for (VrfTables vrfTable : staleVrfTablesMap.values()) {
2967                     Map<String, Uint32> staleFibEntMap = new HashMap<>();
2968                     for (VrfEntry vrfEntry : vrfTable.getVrfEntry().values()) {
2969                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2970                             //Stale marking and cleanup is only meant for the routes learned through BGP.
2971                             continue;
2972                         }
2973                         if (Thread.interrupted()) {
2974                             break;
2975                         }
2976                         totalStaledCount++;
2977                         //Create MAP from staleVrfTablesMap.
2978                         vrfEntry.getRoutePaths().values()
2979                                 .forEach(
2980                                     routePath -> {
2981                                         staleFibEntMap.put(
2982                                                 appendNextHopToPrefix(vrfEntry.getDestPrefix(),
2983                                                         routePath.getNexthopAddress()), routePath.getLabel());
2984                                         fibDSWriter.addEntryToFibMap(
2985                                                 vrfTable.getRouteDistinguisher(),  vrfEntry.getDestPrefix(),
2986                                                 routePath.getNexthopAddress());
2987
2988                                     });
2989                     }
2990                     staledFibEntriesMap.put(vrfTable.getRouteDistinguisher(), staleFibEntMap);
2991                 }
2992             } else {
2993                 LOG.error("createStaleFibMap:: FIBentries.class is not present");
2994             }
2995         } catch (InterruptedException | ExecutionException e) {
2996             LOG.error("createStaleFibMap:: error ", e);
2997         }
2998         LOG.error("created {} staled entries ", totalStaledCount);
2999     }
3000
3001     /*
3002      * BGP config remove scenario, Need to remove all the
3003      * external routes from FIB.
3004      */
3005     public void deleteExternalFibRoutes() {
3006         totalExternalRoutes = 0;
3007         totalExternalMacRoutes = 0;
3008         try {
3009             InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
3010
3011             Optional<FibEntries> fibEntries = SingleTransactionDataBroker.syncReadOptional(dataBroker,
3012                     LogicalDatastoreType.CONFIGURATION, id);
3013             if (fibEntries.isPresent()) {
3014                 if (fibEntries.get().getVrfTables() == null) {
3015                     LOG.error("deleteExternalFibRoutes::getVrfTables is null");
3016                     return;
3017                 }
3018                 Map<VrfTablesKey, VrfTables> staleVrfTablesMap = fibEntries.get().getVrfTables();
3019                 for (VrfTables vrfTable : staleVrfTablesMap.values()) {
3020                     String rd = vrfTable.getRouteDistinguisher();
3021                     if (vrfTable.getVrfEntry() != null) {
3022                         for (VrfEntry vrfEntry : vrfTable.getVrfEntry().values()) {
3023                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
3024                                 //route cleanup is only meant for the routes learned through BGP.
3025                                 continue;
3026                             }
3027                             totalExternalRoutes++;
3028                             fibDSWriter.removeFibEntryFromDS(rd, vrfEntry.getDestPrefix());
3029                         }
3030                     } else if (vrfTable.getMacVrfEntry() != null) {
3031                         for (MacVrfEntry macEntry : vrfTable.getMacVrfEntry().values()) {
3032                             if (RouteOrigin.value(macEntry.getOrigin()) != RouteOrigin.BGP) {
3033                                 //route cleanup is only meant for the routes learned through BGP.
3034                                 continue;
3035                             }
3036                             totalExternalMacRoutes++;
3037                             fibDSWriter.removeMacEntryFromDS(rd, macEntry.getMac());
3038                         }
3039                     }
3040                 }
3041             } else {
3042                 LOG.error("deleteExternalFibRoutes:: FIBentries.class is not present");
3043             }
3044         } catch (InterruptedException | ExecutionException e) {
3045             LOG.error("deleteExternalFibRoutes:: error ", e);
3046         }
3047         LOG.debug("deleted {} fib entries {} mac entries", totalExternalRoutes, totalExternalMacRoutes);
3048     }
3049
3050     public boolean addToRt2TepMap(String rd, String tepIp, String mac, Uint32 l2vni) {
3051         boolean isFirstMacUpdateFromTep = false;
3052         if (rt2TepMap.containsKey(rd)) {
3053             if (rt2TepMap.get(rd).containsKey(tepIp)) {
3054                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and tep-ip {}. No Elan DS write required",
3055                         mac, l2vni, rd, tepIp);
3056                 rt2TepMap.get(rd).get(tepIp).put(mac, l2vni);
3057             } else {
3058                 LOG.debug("RT2 with mac {} l2vni {} from existing rd {} and new tep-ip {}",
3059                         mac, l2vni, rd, tepIp);
3060                 isFirstMacUpdateFromTep = true;
3061                 Map<String, Uint32> macList = new HashMap<>();
3062                 macList.put(mac, l2vni);
3063                 rt2TepMap.get(rd).put(tepIp, macList);
3064             }
3065         } else {
3066             LOG.debug("RT2 with mac {} l2vni {} from new rd {} and tep ip {}",
3067                     mac, l2vni, rd, tepIp);
3068             isFirstMacUpdateFromTep = true;
3069             Map<String, Uint32> macList = new HashMap<>();
3070             macList.put(mac, l2vni);
3071             Map<String, Map<String, Uint32>> tepIpMacMap = new HashMap<>();
3072             tepIpMacMap.put(tepIp, macList);
3073             rt2TepMap.put(rd, tepIpMacMap);
3074         }
3075         return isFirstMacUpdateFromTep;
3076     }
3077
3078     public boolean deleteFromRt2TepMap(String rd, String tepIp, String mac) {
3079         boolean isLastMacUpdateFromTep = false;
3080         LOG.debug("RT2 withdraw with rd {} mac {} tep-ip {} ", rd, mac, tepIp);
3081         if (rt2TepMap.containsKey(rd)) {
3082             if (rt2TepMap.get(rd).containsKey(tepIp)) {
3083                 if (rt2TepMap.get(rd).get(tepIp).containsKey(mac)) {
3084                     LOG.debug("RT2 Withdraw : Removing the mac {} from Map", mac);
3085                     rt2TepMap.get(rd).get(tepIp).remove(mac);
3086                     if (rt2TepMap.get(rd).get(tepIp).isEmpty()) {
3087                         isLastMacUpdateFromTep = true;
3088                         LOG.debug("RT2 Withdraw : Removing the tep-ip {} from Map", tepIp);
3089                         rt2TepMap.get(rd).remove(tepIp);
3090                         if (rt2TepMap.get(rd).isEmpty()) {
3091                             LOG.debug("RT2 Withdraw : Removing the rd {} from Map", rd);
3092                             rt2TepMap.remove(rd);
3093                         }
3094                     }
3095                 }
3096             }
3097         }
3098         return isLastMacUpdateFromTep;
3099     }
3100
3101     public Collection<String> getTepIPs(String rd) {
3102         final Map<String, Map<String, Uint32>> tepIpMap = rt2TepMap.get(rd);
3103         return tepIpMap != null ? tepIpMap.keySet() : Collections.emptyList();
3104     }
3105
3106     public boolean isBgpConnected() {
3107         return (bgpRouter == null) ? false : bgpRouter.isBgpConnected();
3108     }
3109
3110     public long getLastConnectedTS() {
3111         return (bgpRouter == null) ? 0 : bgpRouter.getLastConnectedTS();
3112     }
3113
3114     public long getConnectTS() {
3115         return (bgpRouter == null) ? 0 : bgpRouter.getConnectTS();
3116     }
3117
3118     public long getStartTS() {
3119         return (bgpRouter == null) ? 0 : bgpRouter.getStartTS();
3120     }
3121
3122     public TTransport getTransport() {
3123         return bgpRouter.getTransport();
3124     }
3125
3126     public int getTotalStaledCount() {
3127         return totalStaledCount;
3128     }
3129
3130     public int getTotalCleared() {
3131         return totalCleared;
3132     }
3133
3134     public static List<Neighbors> getNbrList() {
3135         return nbrList;
3136     }
3137
3138     public BgpCounters getBgpCounters() {
3139         return bgpCountersReference.get();
3140     }
3141
3142     private void startBgpCountersTask() {
3143         if (getBgpCounters() == null && bgpCountersReference.compareAndSet(null,
3144                 new BgpCounters(getBgpSdncMipIp(), metricProvider))) {
3145             bgpCountersTask = executor.scheduleAtFixedRate(bgpCountersReference.get(), 0, 120 * 1000,
3146                     TimeUnit.MILLISECONDS);
3147             LOG.info("Bgp Counters task scheduled for every two minutes.");
3148
3149             bgpManager.setQbgpLog(BgpConstants.BGP_DEF_LOG_FILE, BgpConstants.BGP_DEF_LOG_LEVEL);
3150         }
3151     }
3152
3153     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3154             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3155     private void stopBgpCountersTask() {
3156         final BgpCounters bgpCounters = bgpCountersReference.getAndSet(null);
3157         if (bgpCounters != null) {
3158             bgpCountersTask.cancel(true);
3159             bgpCounters.close();
3160         }
3161     }
3162
3163     private void startBgpAlarmsTask() {
3164         if (getBgpAlarms() == null && bgpAlarmsReference.compareAndSet(null, new BgpAlarms(this))) {
3165             bgpAlarmsReference.get().init();
3166             bgpAlarmsTask = executor.scheduleAtFixedRate(bgpAlarmsReference.get(), 0, 60 * 1000, TimeUnit.MILLISECONDS);
3167             LOG.info("Bgp Alarms task scheduled for every minute.");
3168         } else {
3169             LOG.trace("Bgp Alarms task already scheduled for every minute.");
3170         }
3171     }
3172
3173     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3174             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3175     private void stopBgpAlarmsTask() {
3176         final BgpAlarms bgpAlarms = bgpAlarmsReference.getAndSet(null);
3177         if (bgpAlarms != null) {
3178             bgpAlarmsTask.cancel(true);
3179             bgpAlarms.close();
3180         }
3181     }
3182
3183     public BgpAlarms getBgpAlarms() {
3184         return bgpAlarmsReference.get();
3185     }
3186
3187     public void getPeerStatus(String nbrIp, long nbrAsNum) throws
3188             BgpRouterException, TException {
3189         bgpRouter.getPeerStatus(nbrIp, nbrAsNum);
3190     }
3191
3192     private static String appendNextHopToPrefix(String prefix, String nextHop) {
3193         return prefix + ":" + nextHop;
3194     }
3195
3196     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3197             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3198     private static String extractPrefix(String prefixNextHop) {
3199         return prefixNextHop.split(":")[0];
3200     }
3201
3202     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
3203             justification = "https://github.com/spotbugs/spotbugs/issues/811")
3204     private static String extractNextHop(String prefixNextHop) {
3205         return prefixNextHop.split(":")[1];
3206     }
3207
3208     private static String extractMd5Secret(final Neighbors val) {
3209         String md5Secret = null;
3210         TcpSecurityOption tcpSecOpt = val.getTcpSecurityOption();
3211         if (tcpSecOpt != null) {
3212             if (tcpSecOpt instanceof TcpMd5SignatureOption) {
3213                 md5Secret = ((TcpMd5SignatureOption) tcpSecOpt).getTcpMd5SignaturePassword().getValue();
3214             } else { // unknown TcpSecurityOption
3215                 LOG.debug("neighbors  Ignored unknown tcp-security-option of peer {}", val.getAddress().getValue());
3216             }
3217         }
3218         return md5Secret;
3219     } // private method extractMd5Secret
3220
3221     @SuppressWarnings("checkstyle:IllegalCatch")
3222     @Override
3223     public ListenableFuture<RpcResult<InitiateEorOutput>> initiateEor(InitiateEorInput input) {
3224         boolean returnError = false;
3225         String msg = null;
3226         String neighborIp = null;
3227         if (!isBGPEntityOwner()) {
3228             msg = String.format("RPC triggered in Non-EoS Owner");
3229             return Futures.immediateFuture(
3230                     RpcResultBuilder.<InitiateEorOutput>failed().withError(RpcError.ErrorType.APPLICATION,
3231                             msg).build());
3232         }
3233         if (input == null) {
3234             msg = String.format("BGP invalid input for EoR");
3235             LOG.error("Error : {}", msg);
3236             returnError = true;
3237         } else {
3238             neighborIp = input.getNeighborIp();
3239         }
3240         if (eorSupressedDuetoUpgradeFlag.get() == false) {
3241             msg = String.format("EoR triggerd by RBU-RPC call before replay"
3242                     + "of BGP configuration (or) BGP not restarted");
3243             LOG.error("Error : {}", msg);
3244         }
3245         if ("ALL".compareToIgnoreCase(neighborIp) == 0) {
3246             //send EoR for all the neighbor
3247             LOG.error("EoR trigger received to ALL neighbors");
3248             final int numberOfEORRetries = 3;
3249             RetryOnException eorRetry = new RetryOnException(numberOfEORRetries);
3250             do {
3251                 try {
3252                     BgpRouter br = bgpRouter;
3253                     br.sendEOR();
3254                     LOG.debug("RPC: sendEOR {} successful", br);
3255                     break;
3256                 } catch (Exception e) {
3257                     eorRetry.errorOccured();
3258                     LOG.error("Replay:sedEOR() received exception:", e);
3259                 }
3260             } while (eorRetry.shouldRetry());
3261             eorSupressedDuetoUpgradeFlag.set(false);
3262         } else if (InetAddresses.isInetAddress(neighborIp)) {
3263             //send EoR for only one neighbor
3264             msg = String.format("Inidividual neighbors EoR is not supported");
3265             LOG.warn("Error : {}", msg);
3266             returnError = true;
3267         } else {
3268             //error
3269             msg = String.format("RPC: initiateEor: Invalid input ");
3270             LOG.warn("Error : {}", msg);
3271             returnError = true;
3272         }
3273         if (returnError) {
3274             return Futures.immediateFuture(
3275                     RpcResultBuilder.<InitiateEorOutput>failed().withError(RpcError.ErrorType.APPLICATION,
3276                             msg).build());
3277         }
3278         InitiateEorOutput initiateEorOutput =
3279                 new InitiateEorOutputBuilder().setRetVal(0L).build();
3280         return Futures.immediateFuture(RpcResultBuilder.<InitiateEorOutput>success()
3281                 .withResult(initiateEorOutput).build());
3282     }
3283 }