vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.util.concurrent.FutureCallback;
  13 import com.google.common.util.concurrent.Futures;
  14 import com.google.common.util.concurrent.ListenableFuture;
  15 import java.math.BigInteger;
  16 import java.net.Inet4Address;
  17 import java.net.InetAddress;
  18 import java.net.UnknownHostException;
  19 import java.util.ArrayList;
  20 import java.util.Arrays;
  21 import java.util.Collection;
  22 import java.util.Collections;
  23 import java.util.List;
  24 import java.util.concurrent.BlockingQueue;
  25 import java.util.concurrent.Callable;
  26 import java.util.concurrent.ExecutionException;
  27 import java.util.concurrent.Future;
  28 import java.util.concurrent.LinkedBlockingQueue;
  29 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  30 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  31 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  32 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  33 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  34 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  35 import org.opendaylight.genius.mdsalutil.ActionInfo;
  36 import org.opendaylight.genius.mdsalutil.FlowEntity;
  37 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  38 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  39 import org.opendaylight.genius.mdsalutil.MatchInfo;
  40 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  41 import org.opendaylight.genius.mdsalutil.NwConstants;
  42 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  43 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationEth;
  44 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationIp;
  45 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionSetIcmpType;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionSetSourceIp;
  55 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  56 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  57 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  58 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  59 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  60 import org.opendaylight.genius.mdsalutil.matches.MatchIcmpv4;
  61 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  62 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  63 import org.opendaylight.genius.mdsalutil.matches.MatchIpv6Destination;
  64 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  65 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  67 import org.opendaylight.genius.utils.ServiceIndex;
  68 import org.opendaylight.genius.utils.batching.ActionableResource;
  69 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  70 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  71 import org.opendaylight.genius.utils.batching.ResourceHandler;
  72 import org.opendaylight.genius.utils.batching.SubTransaction;
  73 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
  74 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  75 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  76 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  77 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  78 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  79 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  80 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.Tunnel;
  81 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeInputBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeOutput;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroute;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.Vpn;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.VpnKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 135 import org.opendaylight.yangtools.yang.binding.DataObject;
 136 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 137 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 138 import org.opendaylight.yangtools.yang.common.RpcResult;
 139 import org.slf4j.Logger;
 140 import org.slf4j.LoggerFactory;
 141
 142 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener>
 143     implements AutoCloseable, ResourceHandler {
 144
 145     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 146     private static final String FLOWID_PREFIX = "L3.";
 147     private final DataBroker dataBroker;
 148     private final IMdsalApiManager mdsalManager;
 149     private IVpnManager vpnmanager;
 150     private final NexthopManager nextHopManager;
 151     private ItmRpcService itmManager;
 152     private final OdlInterfaceRpcService interfaceManager;
 153     private final IdManagerService idManager;
 154     private static final BigInteger COOKIE_VM_FIB_TABLE = new BigInteger("8000003", 16);
 155     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 156     private static final int LFIB_INTERVPN_PRIORITY = 1;
 157     private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
 158     private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
 159     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 160     List<SubTransaction> transactionObjects;
 161     private static final int PERIODICITY = 500;
 162     private static Integer batchSize;
 163     private static Integer batchInterval;
 164     private static final int BATCH_SIZE = 1000;
 165     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 166     private final ResourceBatchingManager resourceBatchingManager;
 167
 168     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 169                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
 170                             final IdManagerService idManager) {
 171         super(VrfEntry.class, VrfEntryListener.class);
 172         this.dataBroker = dataBroker;
 173         this.mdsalManager = mdsalApiManager;
 174         this.nextHopManager = nexthopManager;
 175         this.interfaceManager = interfaceManager;
 176         this.idManager = idManager;
 177
 178         batchSize = Integer.getInteger("batch.size");
 179         if (batchSize == null) {
 180             batchSize = BATCH_SIZE;
 181         }
 182         batchInterval = Integer.getInteger("batch.wait.time");
 183         if (batchInterval == null) {
 184             batchInterval = PERIODICITY;
 185         }
 186         resourceBatchingManager = ResourceBatchingManager.getInstance();
 187         resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
 188         transactionObjects = new ArrayList<>();
 189     }
 190
 191     public void start() {
 192         LOG.info("{} start", getClass().getSimpleName());
 193         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 194     }
 195
 196     @Override
 197     protected VrfEntryListener getDataTreeChangeListener() {
 198         return VrfEntryListener.this;
 199     }
 200
 201     @Override
 202     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 203         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 204     }
 205
 206     @Override
 207     public DataBroker getResourceBroker() {
 208         return dataBroker;
 209     }
 210
 211     @Override
 212     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 213         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 214         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 215         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} nexthop {} label {}",
 216             rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 217         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 218             createFibEntries(identifier, vrfEntry);
 219         } else {
 220             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 221             actResource.setAction(ActionableResource.CREATE);
 222             actResource.setInstanceIdentifier(identifier);
 223             actResource.setInstance(vrfEntry);
 224             vrfEntryBufferQ.add(actResource);
 225         }
 226         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 227         LOG.info("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
 228             rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 229     }
 230
 231     @Override
 232     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 233         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 234         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 235         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} nexthop {} label {}",
 236             rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 237         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 238             deleteFibEntries(identifier, vrfEntry);
 239         } else {
 240             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 241             actResource.setAction(ActionableResource.DELETE);
 242             actResource.setInstanceIdentifier(identifier);
 243             actResource.setInstance(vrfEntry);
 244             vrfEntryBufferQ.add(actResource);
 245         }
 246         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 247         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
 248             rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 249     }
 250
 251     @Override
 252     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 253         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 254
 255         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 256         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
 257         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
 258             rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 259         // Handle BGP Routes first
 260         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 261             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + update.getDestPrefix());
 262             actResource.setAction(ActionableResource.UPDATE);
 263             actResource.setInstanceIdentifier(identifier);
 264             actResource.setInstance(update);
 265             actResource.setOldInstance(original);
 266             vrfEntryBufferQ.add(actResource);
 267             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 268                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 269             return;
 270         }
 271
 272         // Handle Internal Routes next (ie., STATIC only)
 273         if (FibUtil.isControllerManagedNonInterVpnLinkRoute(RouteOrigin.value(update.getOrigin()))) {
 274             SubnetRoute subnetRoute = update.getAugmentation(SubnetRoute.class);
 275             /* Ignore SubnetRoute entry, as it will be driven by createFibEntries call down below */
 276             if (subnetRoute == null) {
 277                 List<String> origNhList = original.getNextHopAddressList();
 278                 List<String> updateNhList = update.getNextHopAddressList();
 279                 //final SubnetRoute subnetRoute = update.getAugmentation(SubnetRoute.class);
 280                 LOG.info("UPDATE: Original nexthop {} updateNextHop {} ", origNhList, updateNhList);
 281
 282                 // If original VRF Entry had nexthop null , but update VRF Entry
 283                 // has nexthop , route needs to be created on remote Dpns
 284                 if (((origNhList == null) || (origNhList.isEmpty())
 285                     && (updateNhList != null) && (!updateNhList.isEmpty()))) {
 286                     // TODO(vivek): Though ugly, Not handling this code now, as each
 287                     // tep add event will invoke flow addition
 288                     LOG.trace("Original VRF entry NH is null for destprefix {}. This event is IGNORED here.",
 289                         update.getDestPrefix());
 290                     return;
 291                 }
 292
 293                 // If original VRF Entry had valid nexthop , but update VRF Entry
 294                 // has nexthop empty'ed out, route needs to be removed from remote Dpns
 295                 if (((updateNhList == null) || (updateNhList.isEmpty())
 296                     && (origNhList != null) && (!origNhList.isEmpty()))) {
 297                     LOG.trace("Original VRF entry had valid NH for destprefix {}. This event is IGNORED here.",
 298                         update.getDestPrefix());
 299                     return;
 300                 }
 301             }
 302             createFibEntries(identifier, update);
 303             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 304                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 305             return;
 306         }
 307
 308         /* Handl all other route origins */
 309         createFibEntries(identifier, update);
 310
 311         LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 312             rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 313     }
 314
 315     @Override
 316     public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 317                        Object original, Object update, List<SubTransaction> transactionObjects) {
 318         this.transactionObjects = transactionObjects;
 319         if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 320             createFibEntries(tx, identifier, (VrfEntry) update);
 321         }
 322     }
 323
 324     @Override
 325     public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 326                        Object vrfEntry, List<SubTransaction> transactionObjects) {
 327         this.transactionObjects = transactionObjects;
 328         if (vrfEntry instanceof VrfEntry) {
 329             createFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 330         }
 331     }
 332
 333     @Override
 334     public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 335                        Object vrfEntry, List<SubTransaction> transactionObjects) {
 336         this.transactionObjects = transactionObjects;
 337         if (vrfEntry instanceof VrfEntry) {
 338             deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 339         }
 340     }
 341
 342     @Override
 343     public int getBatchSize() {
 344         return batchSize;
 345     }
 346
 347     @Override
 348     public int getBatchInterval() {
 349         return batchInterval;
 350     }
 351
 352     @Override
 353     public LogicalDatastoreType getDatastoreType() {
 354         return LogicalDatastoreType.CONFIGURATION;
 355     }
 356
 357     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 358         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 359
 360         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 361         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 362         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 363             + " has null vpnId!");
 364
 365         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 366         final Long vpnId = vpnInstance.getVpnId();
 367         final String rd = vrfTableKey.getRouteDistinguisher();
 368         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 369         if (subnetRoute != null) {
 370             final long elanTag = subnetRoute.getElantag();
 371             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 372                 rd, vrfEntry.getDestPrefix(), elanTag);
 373             if (vpnToDpnList != null) {
 374                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 375                 dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
 376                     () -> {
 377                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 378                         for (final VpnToDpnList curDpn : vpnToDpnList) {
 379                             if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 380                                 installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId.longValue(),
 381                                     vrfEntry, tx);
 382                             }
 383                         }
 384                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 385                         futures.add(tx.submit());
 386                         return futures;
 387                     });
 388             }
 389             return;
 390         }
 391         // ping responder for router interfaces
 392         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnId, NwConstants.ADD_FLOW)) {
 393             return;
 394         }
 395
 396         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 397
 398         if (vpnToDpnList != null) {
 399             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 400             dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
 401                 () -> {
 402                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 403                     for (VpnToDpnList vpnDpn : vpnToDpnList) {
 404                         if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 405                             if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 406                                 createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 407                                     vrfTableKey, vrfEntry, tx);
 408                             }
 409                         }
 410                     }
 411                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 412                     futures.add(tx.submit());
 413                     return futures;
 414                 });
 415         }
 416
 417         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 418         if (optVpnUuid.isPresent()) {
 419             Optional<InterVpnLinkDataComposite> optInterVpnLink =
 420                 InterVpnLinkCache.getInterVpnLinkByVpnId(optVpnUuid.get());
 421             LOG.debug("InterVpnLink {} found in Cache: {}", optVpnUuid.get(), optInterVpnLink.isPresent());
 422             if (optInterVpnLink.isPresent()) {
 423                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
 424                 String vpnUuid = optVpnUuid.get();
 425                 String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
 426                 if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 427                     // This is an static route that points to the other endpoint of an InterVpnLink
 428                     // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 429                     installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 430                     installInterVpnRouteInLFib(rd, vrfEntry);
 431                 }
 432             }
 433         }
 434     }
 435
 436
 437     /*
 438       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 439       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 440       provided by ResourceBatchingManager
 441      */
 442     private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
 443                                   final VrfEntry vrfEntry) {
 444         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 445
 446         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 447         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 448         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 449             + " has null vpnId!");
 450
 451         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 452         final String rd = vrfTableKey.getRouteDistinguisher();
 453         if (vpnToDpnList != null) {
 454             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 455                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 456                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 457                 }
 458             }
 459         }
 460     }
 461
 462     private boolean originMustBeLeaked(InterVpnLink ivpnLink, RouteOrigin routeOrigin) {
 463         return routeOrigin == RouteOrigin.BGP && ivpnLink.isBgpRoutesLeaking()
 464             || routeOrigin == RouteOrigin.STATIC && ivpnLink.isStaticRoutesLeaking()
 465             || routeOrigin == RouteOrigin.CONNECTED && ivpnLink.isConnectedRoutesLeaking();
 466     }
 467
 468     // FIXME: Refactoring needed here.
 469     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 470     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 471                                    int addOrRemove) {
 472         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 473         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 474
 475         String rd = vrfTableKey.getRouteDistinguisher();
 476         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 477         if (vpnInstance == null) {
 478             LOG.error("VPN Instance not available for route with prefix {} label {} nextHop {} RD {}. Returning...",
 479                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
 480             return;
 481         }
 482         String vpnUuid = vpnInstance.getVpnInstanceName();
 483         Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(),
 484             "Could not find suitable VPN UUID for Route-Distinguisher=" + rd);
 485
 486         // if the new vrfEntry has been learned by Quagga BGP, its necessary to check if it's
 487         // there an interVpnLink for the involved vpn in order to make learn the new route to
 488         // the other part of the inter-vpn-link.
 489
 490         // For leaking, we need the InterVpnLink to be active. For removal, we just need a InterVpnLink.
 491         Optional<InterVpnLink> interVpnLink =
 492             (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(dataBroker, rd)
 493                 : FibUtil.getInterVpnLinkByRd(dataBroker, rd);
 494         if (!interVpnLink.isPresent()) {
 495             LOG.debug("Could not find an InterVpnLink for Route-Distinguisher={}", rd);
 496             return;
 497         }
 498
 499         // Ok, at this point everything is ready for the leaking/removal... but should it be performed?
 500         // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled.
 501         boolean proceed =
 502             addOrRemove == NwConstants.DEL_FLOW || originMustBeLeaked(interVpnLink.get(),
 503                 RouteOrigin.value(vrfEntry.getOrigin()));
 504
 505         if (proceed) {
 506             boolean isVpnFirstEndpoint = interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid);
 507
 508             String theOtherVpnId = isVpnFirstEndpoint ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
 509                 : interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue();
 510             String dstVpnRd = FibUtil.getVpnRd(dataBroker, theOtherVpnId);
 511             String endpointIp = isVpnFirstEndpoint ? interVpnLink.get().getFirstEndpoint().getIpAddress().getValue()
 512                 : interVpnLink.get().getSecondEndpoint().getIpAddress().getValue();
 513
 514             InstanceIdentifier<VrfEntry> vrfEntryIidInOtherVpn =
 515                 InstanceIdentifier.builder(FibEntries.class)
 516                     .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
 517                     .child(VrfEntry.class, new VrfEntryKey(vrfEntry.getDestPrefix()))
 518                     .build();
 519             if (addOrRemove == NwConstants.ADD_FLOW) {
 520                 LOG.debug("Leaking route (destination={}, nexthop={}) from Vrf={} to Vrf={}",
 521                     vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), rd, dstVpnRd);
 522                 String key = rd + FibConstants.SEPARATOR + vrfEntry.getDestPrefix();
 523                 long label = FibUtil.getUniqueId(idManager, FibConstants.VPN_IDPOOL_NAME, key);
 524                 VrfEntry newVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(Arrays.asList(endpointIp))
 525                     .setLabel(label)
 526                     .setOrigin(RouteOrigin.INTERVPN.getValue())
 527                     .build();
 528                 MDSALUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
 529                     vrfEntryIidInOtherVpn, newVrfEntry);
 530             } else {
 531                 LOG.debug("Removing leaked vrfEntry={}", vrfEntryIidInOtherVpn.toString());
 532                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn);
 533             }
 534         }
 535     }
 536
 537     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 538         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 539         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 540         prefixBuilder.setDpnId(lri.getDpnId());
 541         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 542         prefixBuilder.setIpAddress(lri.getPrefix());
 543         // Increment the refCount here
 544         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 545             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 546         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 547         if (!isPresentInList) {
 548             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 549             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 550             vpnInstanceNames.add(vpnInstanceName);
 551             builder.setVpnInstanceList(vpnInstanceNames);
 552             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(),
 553                 FibUtil.DEFAULT_CALLBACK);
 554         } else {
 555             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 556         }
 557         return prefixBuilder.build();
 558     }
 559
 560     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 561                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 562         Boolean wrTxPresent = true;
 563         if (tx == null) {
 564             wrTxPresent = false;
 565             tx = dataBroker.newWriteOnlyTransaction();
 566         }
 567         synchronized (vrfEntry.getLabel().toString().intern()) {
 568             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 569             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
 570                 && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 571
 572                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 573                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 574                         FibUtil.getVpnInstanceOpData(dataBroker, rd);
 575                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
 576                         String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 577                         if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 578                             updateVpnReferencesInLri(lri, vpnInstanceName, false);
 579                         }
 580                     }
 581                 }
 582                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 583                     vrfEntry.getLabel(), lri.getVpnInterfaceName(), lri.getDpnId());
 584             }
 585         }
 586         final List<InstructionInfo> instructions = new ArrayList<>();
 587         BigInteger subnetRouteMeta = ((BigInteger.valueOf(elanTag)).shiftLeft(32))
 588             .or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 589         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 590         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 591         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 592
 593         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 594             List<ActionInfo> actionsInfos = new ArrayList<>();
 595             // reinitialize instructions list for LFIB Table
 596             final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 597
 598             actionsInfos.add(new ActionPopMpls());
 599             LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 600             LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 601                 MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 602             LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 603
 604             makeLFibTableEntry(dpnId, vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 605                 NwConstants.ADD_FLOW, tx);
 606         }
 607         if (!wrTxPresent) {
 608             tx.submit();
 609         }
 610     }
 611
 612     /*
 613      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 614      * LportDispatcher table (via table 80)
 615      */
 616     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 617         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 618         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 619         // packet is commuted from Vpn2 to Vpn1.
 620         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
 621         if (!vpnNameOpc.isPresent()) {
 622             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 623             return;
 624         }
 625
 626         String vpnName = vpnNameOpc.get();
 627         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
 628         boolean interVpnLinkFound = false;
 629         for (InterVpnLink interVpnLink : interVpnLinks) {
 630             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 631             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 632                 && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 633             if (vpnIs1stEndpoint || vpnIs2ndEndpoint) {
 634                 interVpnLinkFound = true;
 635
 636                 Optional<InterVpnLinkState> vpnLinkState =
 637                     FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 638                 if (!vpnLinkState.isPresent()
 639                     || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active)) {
 640                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 641                         interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 642                         interVpnLink.getSecondEndpoint().getVpnUuid().getValue());
 643                     return;
 644                 }
 645
 646                 List<BigInteger> targetDpns = vpnIs1stEndpoint ? vpnLinkState.get().getFirstEndpointState().getDpId()
 647                     : vpnLinkState.get().getSecondEndpointState().getDpId();
 648                 Long lportTag = vpnIs1stEndpoint ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 649                     : vpnLinkState.get().getFirstEndpointState().getLportTag();
 650
 651                 LOG.trace("Installing flow in LFIB table for interVpnLink {}", interVpnLink.getName());
 652
 653                 for (BigInteger dpId : targetDpns) {
 654                     List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls());
 655
 656                     List<InstructionInfo> instructions =
 657                         Arrays.asList(new InstructionApplyActions(actionsInfos),
 658                             new InstructionWriteMetadata(
 659                                 MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 660                                     ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 661                                         NwConstants.L3VPN_SERVICE_INDEX)),
 662                                 MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 663                             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 664
 665                     LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for "
 666                         + "InterVpnLink {} in LFIB",
 667                         vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(),
 668                         dpId, interVpnLink.getName());
 669
 670                     makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY,
 671                         NwConstants.ADD_FLOW, null);
 672                 }
 673
 674                 break;
 675             }
 676         }
 677
 678         if (!interVpnLinkFound) {
 679             LOG.warn("VrfEntry=[prefix={} label={} nexthop={}] for VPN {} has origin INTERVPN but "
 680                 + "no InterVpnLink could be found",
 681                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
 682         }
 683     }
 684
 685
 686     /*
 687      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 688      */
 689     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 690                                                final VrfEntry vrfEntry, long vpnTag) {
 691         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 692         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 693             && vrfEntry.getNextHopAddressList().size() == 1);
 694         String destination = vrfEntry.getDestPrefix();
 695         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 696         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 697
 698         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 699         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 700         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 701         if (interVpnLink.getState().or(State.Error) != State.Active) {
 702             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 703                 destination, nextHop, interVpnLinkName);
 704             return;
 705         }
 706
 707         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 708         if (!optOtherEndpointLportTag.isPresent()) {
 709             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 710                 vpnUuid, interVpnLinkName);
 711             return;
 712         }
 713
 714         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 715         if (targetDpns.isEmpty()) {
 716             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 717                 vpnUuid, interVpnLinkName);
 718             return;
 719         }
 720
 721         String[] values = destination.split("/");
 722         String destPrefixIpAddress = values[0];
 723         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 724
 725         List<MatchInfo> matches = new ArrayList<>();
 726         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 727         matches.add(MatchEthernetType.IPV4);
 728
 729         if (prefixLength != 0) {
 730             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 731         }
 732
 733         List<Instruction> instructions =
 734             Arrays.asList(new InstructionWriteMetadata(
 735                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 736                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 737                             .L3VPN_SERVICE_INDEX)),
 738                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 739                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 740
 741         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 742         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 743         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 744             COOKIE_VM_FIB_TABLE, matches, instructions);
 745
 746         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 747             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 748
 749         for (BigInteger dpId : targetDpns) {
 750
 751             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nextHop={}] dpn {} for InterVpnLink {} in FIB",
 752                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(),
 753                 dpId, interVpnLink.getInterVpnLinkName());
 754
 755             mdsalManager.installFlow(dpId, flowEntity);
 756         }
 757     }
 758
 759
 760     // TODO Clean up the exception handling
 761     @SuppressWarnings("checkstyle:IllegalCatch")
 762     private <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 763                                                     InstanceIdentifier<T> path) {
 764
 765         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 766
 767         Optional<T> result = Optional.absent();
 768         try {
 769             result = tx.read(datastoreType, path).get();
 770         } catch (Exception e) {
 771             throw new RuntimeException(e);
 772         }
 773
 774         return result;
 775     }
 776
 777     private List<BigInteger> getDpnIdForPrefix(DataBroker broker, Long vpnId, String rd, VrfEntry vrfEntry) {
 778         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 779         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, vrfEntry.getDestPrefix());
 780
 781         if (localNextHopInfo == null) {
 782             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 783             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 784             if (extraRoute != null) {
 785                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 786                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 787                     if (nextHopIp != null) {
 788                         localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, nextHopIp + "/32");
 789                         if (localNextHopInfo != null) {
 790                             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 791                         }
 792                     }
 793                 }
 794             }
 795         } else {
 796             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 797         }
 798
 799         return returnLocalDpnId;
 800     }
 801
 802     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 803         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 804         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
 805         String localNextHopIP = vrfEntry.getDestPrefix();
 806
 807         if (localNextHopInfo == null) {
 808             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 809             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 810             if (extraRoute != null) {
 811                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 812                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 813                     if (nextHopIp != null) {
 814                         localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32");
 815                         localNextHopIP = nextHopIp + "/32";
 816                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 817                             rd, vrfEntry, vpnId);
 818                         returnLocalDpnId.add(dpnId);
 819                     }
 820                 }
 821             }
 822             if (localNextHopInfo == null) {
 823             /* imported routes case */
 824                 synchronized (vrfEntry.getLabel().toString().intern()) {
 825                     LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 826                     if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
 827                         && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 828                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 829                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 830                                 FibUtil.getVpnInstanceOpData(dataBroker, rd);
 831                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 832                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 833                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 834                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 835                                     localNextHopIP = lri.getPrefix();
 836                                 } else {
 837                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 838                                     localNextHopIP = lri.getPrefix();
 839                                 }
 840                             }
 841                             if (localNextHopInfo != null) {
 842                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 843                                     vrfEntry.getLabel(), localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 844                                 BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 845                                     vpnId, rd, vrfEntry, lri.getParentVpnid());
 846                                 returnLocalDpnId.add(dpnId);
 847                             }
 848                         }
 849                     }
 850                 }
 851             }
 852         } else {
 853             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 854             returnLocalDpnId.add(dpnId);
 855         }
 856
 857         return returnLocalDpnId;
 858     }
 859
 860     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 861                                                 final Long vpnId, final String rd,
 862                                                 final VrfEntry vrfEntry, Long parentVpnId) {
 863         if (localNextHopInfo != null) {
 864             final BigInteger dpnId = localNextHopInfo.getDpnId();
 865             if (!isVpnPresentInDpn(rd, dpnId)) {
 866                 LOG.error("The vpnName with vpnId {} rd {} is not available on dpn {}", vpnId, rd, dpnId.toString());
 867                 return BigInteger.ZERO;
 868             }
 869
 870             final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId,
 871                 localNextHopInfo.getVpnInterfaceName(), localNextHopIP, vrfEntry.getDestPrefix());
 872             if (groupId == 0) {
 873                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 874                     vrfEntry.getDestPrefix(), rd, localNextHopInfo.getVpnInterfaceName(), dpnId.toString());
 875                 return BigInteger.ZERO;
 876             }
 877             final List<InstructionInfo> instructions = Collections.singletonList(
 878                 new InstructionApplyActions(
 879                     Collections.singletonList(new ActionGroup(groupId))));
 880             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 881                 new InstructionApplyActions(
 882                     Arrays.asList(new ActionPopMpls(), new ActionGroup(groupId))));
 883             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 884                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 885                     dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
 886             } else {
 887                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. "
 888                     + "LFib and Terminating table entries will not be created.",
 889                     rd, vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpnId);
 890             }
 891             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 892             dataStoreCoordinator.enqueueJob("FIB-" + vpnId.toString()
 893                     + "-" + dpnId.toString() + "-" + vrfEntry.getDestPrefix(),
 894                 () -> {
 895                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 896                     makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 897                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 898                         makeLFibTableEntry(dpnId, vrfEntry.getLabel(), lfibinstructions,
 899                             DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 900                         makeTunnelTableEntry(dpnId, vrfEntry.getLabel(), groupId, tx);
 901                     }
 902                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 903                     futures.add(tx.submit());
 904                     return futures;
 905                 });
 906             return dpnId;
 907         }
 908         return BigInteger.ZERO;
 909     }
 910
 911     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId) {
 912         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 913         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 914         if (dpnInVpn.isPresent()) {
 915             return true;
 916         }
 917         return false;
 918     }
 919
 920     private LabelRouteInfo getLabelRouteInfo(Long label) {
 921         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 922             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 923         Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 924         if (opResult.isPresent()) {
 925             return opResult.get();
 926         }
 927         return null;
 928     }
 929
 930     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName) {
 931         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 932         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 933             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 934         if (lri == null) {
 935             return true;
 936         }
 937         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 938             ? lri.getVpnInstanceList() : new ArrayList<>();
 939         if (vpnInstancesList.contains(vpnInstanceName)) {
 940             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 941             vpnInstancesList.remove(vpnInstanceName);
 942         }
 943         if (vpnInstancesList.size() == 0) {
 944             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 945             FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 946             return true;
 947         } else {
 948             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 949             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 950             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(),
 951                 FibUtil.DEFAULT_CALLBACK);
 952         }
 953         return false;
 954     }
 955
 956     private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 957                                       WriteTransaction tx) {
 958         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 959
 960         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 961
 962         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 963             dpId, label, groupId);
 964     }
 965
 966     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 967                                                 WriteTransaction tx) {
 968         List<MatchInfo> mkMatches = new ArrayList<>();
 969
 970         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 971             destDpId, label, actionsInfos);
 972
 973         // Matching metadata
 974         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 975         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 976
 977         List<InstructionInfo> mkInstructions = new ArrayList<>();
 978         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 979
 980         FlowEntity terminatingServiceTableFlowEntity =
 981             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 982             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
 983                 String.format("%s:%d", "TST Flow Entry ", label),
 984             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
 985
 986         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
 987
 988         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 989
 990         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 991         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 992             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 993             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
 994             .child(Flow.class, flowKey).build();
 995         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(), true);
 996     }
 997
 998     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 999         FlowEntity flowEntity;
1000         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1001         List<MatchInfo> mkMatches = new ArrayList<>();
1002         // Matching metadata
1003         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1004         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1005             NwConstants.INTERNAL_TUNNEL_TABLE,
1006             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1007             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1008             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1009         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
1010         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1011         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1012             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1013             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1014
1015         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1016         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1017     }
1018
1019     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1020         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1021         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1022         String localNextHopIP = vrfEntry.getDestPrefix();
1023
1024         if (localNextHopInfo == null) {
1025             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1026             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1027             if (extraRoute != null) {
1028                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1029                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1030                     if (nextHopIp != null) {
1031                         localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
1032                         localNextHopIP = nextHopIp + "/32";
1033                         BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1034                             vpnId, rd, vrfEntry, true /*isExtraRoute*/);
1035                         if (!dpnId.equals(BigInteger.ZERO)) {
1036                             returnLocalDpnId.add(dpnId);
1037                         }
1038                     }
1039                 }
1040             }
1041
1042             if (localNextHopInfo == null) {
1043               /* Imported VRF entry */
1044                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1045                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1046                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1047                     VpnNexthopBuilder vpnNexthopBuilder = new VpnNexthopBuilder();
1048                     vpnNexthopBuilder.setDpnId(lri.getDpnId());
1049                     BigInteger dpnId = checkDeleteLocalFibEntry(vpnNexthopBuilder.build(), localNextHopIP,
1050                         vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1051                     if (!dpnId.equals(BigInteger.ZERO)) {
1052                         returnLocalDpnId.add(dpnId);
1053                     }
1054                 }
1055             }
1056
1057         } else {
1058             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1059                 vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1060             if (!dpnId.equals(BigInteger.ZERO)) {
1061                 returnLocalDpnId.add(dpnId);
1062             }
1063         }
1064
1065         return returnLocalDpnId;
1066     }
1067
1068     private BigInteger checkDeleteLocalFibEntry(VpnNexthop localNextHopInfo, final String localNextHopIP,
1069                                                 final Long vpnId, final String rd,
1070                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
1071         if (localNextHopInfo != null) {
1072             final BigInteger dpnId = localNextHopInfo.getDpnId();
1073             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1074             dataStoreCoordinator.enqueueJob("FIB-" + vpnId.toString() + "-"
1075                     + dpnId.toString() + "-" + vrfEntry.getDestPrefix(),
1076                 () -> {
1077                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1078                     makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
1079                         NwConstants.DEL_FLOW, tx);
1080                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1081                         makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null /* instructions */,
1082                             DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1083                         removeTunnelTableEntry(dpnId, vrfEntry.getLabel(), tx);
1084                     }
1085                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1086                     futures.add(tx.submit());
1087                     return futures;
1088                 });
1089             //TODO: verify below adjacency call need to be optimized (?)
1090             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1091             return dpnId;
1092         }
1093         return BigInteger.ZERO;
1094     }
1095
1096     private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
1097         return InstanceIdentifier.builder(VpnToExtraroute.class)
1098             .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
1099                 new ExtrarouteKey(ipPrefix)).build();
1100     }
1101
1102     private Extraroute getVpnToExtraroute(String rd, String ipPrefix) {
1103         Optional<Extraroute> extraRouteInfo =
1104             FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getVpnToExtrarouteIdentifier(rd, ipPrefix));
1105         return extraRouteInfo.isPresent() ? extraRouteInfo.get() : null;
1106
1107     }
1108
1109     private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
1110         try {
1111             Future<RpcResult<GetTunnelTypeOutput>> result = interfaceManager.getTunnelType(
1112                 new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
1113             RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
1114             if (!rpcResult.isSuccessful()) {
1115                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
1116             } else {
1117                 return rpcResult.getResult().getTunnelType();
1118             }
1119
1120         } catch (InterruptedException | ExecutionException e) {
1121             LOG.warn("Exception when getting tunnel interface Id for tunnel type", e);
1122         }
1123
1124         return null;
1125
1126     }
1127
1128     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1129                                       final VrfEntry vrfEntry, WriteTransaction tx) {
1130         Boolean wrTxPresent = true;
1131         if (tx == null) {
1132             wrTxPresent = false;
1133             tx = dataBroker.newWriteOnlyTransaction();
1134         }
1135         String rd = vrfTableKey.getRouteDistinguisher();
1136         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1137             vrfEntry.getDestPrefix(), rd, remoteDpnId);
1138
1139         List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1140         if (adjacencyResults.isEmpty()) {
1141             LOG.error("Could not get interface for nexthop: {} in vpn {}",
1142                 vrfEntry.getNextHopAddressList(), rd);
1143             LOG.warn("Failed to add Route: {} in vpn: {}",
1144                 vrfEntry.getDestPrefix(), rd);
1145             return;
1146         }
1147
1148         for (AdjacencyResult adjacencyResult : adjacencyResults) {
1149             List<ActionInfo> actionInfos = new ArrayList<>();
1150             String egressInterface = adjacencyResult.getInterfaceName();
1151             if (Tunnel.class.equals(adjacencyResult.getInterfaceType())) {
1152                 addTunnelInterfaceActions(egressInterface, vpnId, vrfEntry, actionInfos);
1153             } else {
1154                 addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1155             }
1156             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(egressInterface);
1157             if (egressActions.isEmpty()) {
1158                 LOG.error("Failed to retrieve egress action for prefix {} nextHop {} interface {}. "
1159                     + "Aborting remote FIB entry creation.",
1160                     vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), egressInterface);
1161                 return;
1162             }
1163             actionInfos.addAll(egressActions);
1164             List<InstructionInfo> instructions = new ArrayList<>();
1165             instructions.add(new InstructionApplyActions(actionInfos));
1166             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1167         }
1168         if (!wrTxPresent) {
1169             tx.submit();
1170         }
1171         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1172     }
1173
1174     private void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, List<ActionInfo> actionInfos) {
1175         String ipPrefix = vrfEntry.getDestPrefix();
1176         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, ipPrefix);
1177         if (prefixInfo == null) {
1178             LOG.debug("No prefix info found for prefix {}", ipPrefix);
1179             return;
1180         }
1181
1182         String ifName = prefixInfo.getVpnInterfaceName();
1183         if (ifName == null) {
1184             LOG.warn("Failed to get VPN interface for prefix {}", ipPrefix);
1185             return;
1186         }
1187
1188         String macAddress = FibUtil.getMacAddressFromPrefix(dataBroker, ifName, ipPrefix);
1189         if (macAddress == null) {
1190             LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix);
1191             return;
1192         }
1193
1194         actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(), new MacAddress(macAddress)));
1195     }
1196
1197     private void addTunnelInterfaceActions(String tunnelInterface, long vpnId, VrfEntry vrfEntry,
1198                                            List<ActionInfo> actionInfos) {
1199         Class<? extends TunnelTypeBase> tunnelType = getTunnelType(tunnelInterface);
1200         if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
1201             LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1202             actionInfos.add(new ActionPushMpls());
1203             actionInfos.add(new ActionSetFieldMplsLabel(vrfEntry.getLabel()));
1204             actionInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
1205         } else {
1206             int label = vrfEntry.getLabel().intValue();
1207             BigInteger tunnelId;
1208             // FIXME vxlan vni bit set is not working properly with OVS.need to
1209             // revisit
1210             if (tunnelType.equals(TunnelTypeVxlan.class)) {
1211                 tunnelId = BigInteger.valueOf(label);
1212             } else {
1213                 tunnelId = BigInteger.valueOf(label);
1214             }
1215
1216             LOG.debug("adding set tunnel id action for label {}", label);
1217             actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
1218             addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1219         }
1220     }
1221
1222     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1223         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1224         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1225         if (dpnInVpn.isPresent()) {
1226             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1227             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1228
1229             if (vpnInterfaces.remove(currVpnInterface)) {
1230                 if (vpnInterfaces.isEmpty()) {
1231                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1232                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1233                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
1234                 } else {
1235                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1236                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1237                         VpnInterfaces.class,
1238                         new VpnInterfacesKey(intfName)));
1239                 }
1240             }
1241         }
1242     }
1243
1244     private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
1245     /* Get interface info from prefix to interface mapping;
1246         Use the interface info to get the corresponding vpn interface op DS entry,
1247         remove the adjacency corresponding to this fib entry.
1248         If adjacency removed is the last adjacency, clean up the following:
1249          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1250          - prefix to interface entry
1251          - vpn interface op DS
1252      */
1253         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1254         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1255         Extraroute extraRoute = null;
1256         if (prefixInfo == null) {
1257             extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1258             if (extraRoute != null) {
1259                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1260                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1261
1262                     if (nextHopIp != null) {
1263                         prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32");
1264                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1265                     }
1266                 }
1267             }
1268             if (prefixInfo == null) {
1269                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1270                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1271                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1272                     PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1273                     prefixBuilder.setDpnId(lri.getDpnId());
1274                     prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1275                     prefixBuilder.setIpAddress(lri.getPrefix());
1276                     prefixInfo = prefixBuilder.build();
1277                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1278                         vrfEntry.getLabel(), prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1279                     checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1280                 }
1281             }
1282         } else {
1283             checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1284         }
1285     }
1286
1287     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1288                                           final VrfEntry vrfEntry, final Extraroute extraRoute) {
1289
1290         if (prefixInfo == null) {
1291             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1292             return; //Don't have any info for this prefix (shouldn't happen); need to return
1293         }
1294
1295         String ifName = prefixInfo.getVpnInterfaceName();
1296         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1297         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1298             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1299     }
1300
1301     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1302         Prefixes prefixInfo;
1303         Long vpnId;
1304         String rd;
1305         VrfEntry vrfEntry;
1306         Extraroute extraRoute;
1307
1308         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1309                                          final VrfEntry vrfEntry, final Extraroute extraRoute) {
1310             this.prefixInfo = prefixInfo;
1311             this.vpnId = vpnId;
1312             this.rd = rd;
1313             this.vrfEntry = vrfEntry;
1314             this.extraRoute = extraRoute;
1315         }
1316
1317         @Override
1318         public List<ListenableFuture<Void>> call() throws Exception {
1319             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1320             // to call the respective helpers.
1321
1322             //First Cleanup LabelRouteInfo
1323             synchronized (vrfEntry.getLabel().toString().intern()) {
1324                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1325                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1326                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1327                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1328                         FibUtil.getVpnInstanceOpData(dataBroker, rd);
1329                     String vpnInstanceName = "";
1330                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1331                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1332                     }
1333                     boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName);
1334                     if (lriRemoved) {
1335                         String parentRd = lri.getParentVpnRd();
1336                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1337                             FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1338                     }
1339                 } else {
1340                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1341                         FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1342                 }
1343             }
1344             String ifName = prefixInfo.getVpnInterfaceName();
1345             Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1346                 FibUtil.getVpnInterfaceIdentifier(ifName));
1347             if (optvpnInterface.isPresent()) {
1348                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1349                 if (vpnId != associatedVpnId) {
1350                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1351                         vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1352                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1353                     return null;
1354                 } else {
1355                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1356                         vrfEntry.getDestPrefix(), associatedVpnId);
1357                 }
1358             }
1359             if (extraRoute != null) {
1360                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1361                     FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1362             }
1363             Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1364                 FibUtil.getAdjListPath(ifName));
1365             int numAdj = 0;
1366             if (optAdjacencies.isPresent()) {
1367                 numAdj = optAdjacencies.get().getAdjacency().size();
1368             }
1369             //remove adjacency corr to prefix
1370             if (numAdj > 1) {
1371                 LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
1372                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1373                     FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1374             }
1375             if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
1376                 //clean up the vpn interface from DpnToVpn list
1377                 LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1378                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1379                     FibUtil.getVpnInterfaceIdentifier(ifName));
1380             }
1381             return null;
1382         }
1383     }
1384
1385     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1386         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1387         final String rd = vrfTableKey.getRouteDistinguisher();
1388         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1389         if (vpnInstance == null) {
1390             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1391             return;
1392         }
1393         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1394         long elanTag = 0L;
1395         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1396         if (subnetRoute != null) {
1397             elanTag = subnetRoute.getElantag();
1398             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1399                 rd, vrfEntry.getDestPrefix(), elanTag);
1400             if (vpnToDpnList != null) {
1401                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1402                 dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
1403                     () -> {
1404                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1405
1406                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1407
1408                             makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1409                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1410                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1411                                 makeLFibTableEntry(curDpn.getDpnId(), vrfEntry.getLabel(), null,
1412                                     DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1413                             }
1414                         }
1415                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1416                         futures.add(tx.submit());
1417                         return futures;
1418                     });
1419             }
1420             synchronized (vrfEntry.getLabel().toString().intern()) {
1421                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1422                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1423                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1424                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1425                         FibUtil.getVpnInstanceOpData(dataBroker, rd);
1426                     String vpnInstanceName = "";
1427                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1428                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1429                     }
1430                     boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName);
1431                     if (lriRemoved) {
1432                         String parentRd = lri.getParentVpnRd();
1433                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1434                             FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1435                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as "
1436                             + "labelRouteInfo cleared", vrfEntry.getLabel(), rd,
1437                             vrfEntry.getDestPrefix());
1438                     }
1439                 } else {
1440                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1441                         FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1442                     LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1443                         vrfEntry.getLabel(), rd, vrfEntry.getDestPrefix());
1444                 }
1445             }
1446             return;
1447         }
1448         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnInstance.getVpnId(), NwConstants.DEL_FLOW)) {
1449             return;
1450         }
1451
1452         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1453             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1454         if (vpnToDpnList != null) {
1455             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1456             dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
1457                 () -> {
1458                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1459
1460                     if (localDpnIdList.size() <= 0) {
1461                         for (VpnToDpnList curDpn : vpnToDpnList) {
1462                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1463                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1464                                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1465                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1466                                 }
1467                             } else {
1468                                 deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1469                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1470                             }
1471                         }
1472                     } else {
1473                         for (BigInteger localDpnId : localDpnIdList) {
1474                             for (VpnToDpnList curDpn : vpnToDpnList) {
1475                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1476                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1477                                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1478                                             deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1479                                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1480                                         }
1481                                     } else {
1482                                         deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1483                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1484                                     }
1485                                 }
1486                             }
1487                         }
1488                     }
1489                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1490                     futures.add(tx.submit());
1491                     return futures;
1492                 });
1493         }
1494
1495         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1496         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1497         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1498
1499         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1500         // of the interVpnLink.
1501         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(this.dataBroker, rd);
1502         if (optVpnUuid.isPresent()) {
1503             String vpnUuid = optVpnUuid.get();
1504             List<String> routeNexthoplist = vrfEntry.getNextHopAddressList();
1505             if (routeNexthoplist.isEmpty()) {
1506                 LOG.trace("NextHopList is empty for VrfEntry {}", vrfEntry);
1507                 return;
1508             }
1509             String routeNexthop = routeNexthoplist.get(0);
1510             Optional<InterVpnLinkDataComposite> optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1511             if (optInterVpnLink.isPresent()) {
1512                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1513                 if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1514                     // This is route that points to the other endpoint of an InterVpnLink
1515                     // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
1516                     removeInterVPNLinkRouteFlows(interVpnLink.getInterVpnLinkName(),
1517                         interVpnLink.isFirstEndpointVpnName(rd),
1518                         vrfEntry);
1519                 }
1520             }
1521         }
1522
1523     }
1524
1525     /*
1526       Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1527       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1528       provided by ResourceBatchingManager
1529      */
1530     private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
1531                                   final VrfEntry vrfEntry) {
1532         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1533
1534         final String rd = vrfTableKey.getRouteDistinguisher();
1535         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1536         if (vpnInstance == null) {
1537             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1538             return;
1539         }
1540         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1541         if (vpnToDpnList != null) {
1542             for (VpnToDpnList curDpn : vpnToDpnList) {
1543                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1544                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(),
1545                         vrfTableKey, vrfEntry, writeTx);
1546                 }
1547             }
1548         }
1549     }
1550
1551     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1552                                   final long vpnId, final VrfTablesKey vrfTableKey,
1553                                   final VrfEntry vrfEntry, WriteTransaction tx) {
1554
1555         Boolean wrTxPresent = true;
1556         if (tx == null) {
1557             wrTxPresent = false;
1558             tx = dataBroker.newWriteOnlyTransaction();
1559         }
1560
1561         LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
1562             vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
1563         String rd = vrfTableKey.getRouteDistinguisher();
1564
1565         if (localDpnId != null) {
1566             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1567             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1568             return;
1569         }
1570
1571         // below two reads are kept as is, until best way is found to identify dpnID
1572         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1573         Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1574
1575         if (localNextHopInfo == null && extraRoute != null) {
1576             // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1577             for (String nextHopIp : extraRoute.getNexthopIpList()) {
1578                 localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
1579                 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1580             }
1581         } else {
1582             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1583         }
1584         if (!wrTxPresent) {
1585             tx.submit();
1586         }
1587     }
1588
1589     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1590                                            VrfEntry vrfEntry, String rd, WriteTransaction tx) {
1591         boolean isRemoteRoute = true;
1592         if (localNextHopInfo != null) {
1593             isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId());
1594         }
1595         if (isRemoteRoute) {
1596             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1597             return true;
1598         } else {
1599             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}",
1600                 rd, vrfEntry.getDestPrefix(), remoteDpnId);
1601             return false;
1602         }
1603     }
1604
1605     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry, String rd, WriteTransaction tx) {
1606         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1607         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1608     }
1609
1610     private long get(byte[] rawIpAddress) {
1611         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1612             + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1613     }
1614
1615     private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1616                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1617         Boolean wrTxPresent = true;
1618         if (tx == null) {
1619             wrTxPresent = false;
1620             tx = dataBroker.newWriteOnlyTransaction();
1621         }
1622
1623         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1624         String[] values = vrfEntry.getDestPrefix().split("/");
1625         String ipAddress = values[0];
1626         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1627         if (addOrRemove == NwConstants.ADD_FLOW) {
1628             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1629         } else {
1630             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1631         }
1632         InetAddress destPrefix;
1633         try {
1634             destPrefix = InetAddress.getByName(ipAddress);
1635         } catch (UnknownHostException e) {
1636             LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1637             return;
1638         }
1639
1640         List<MatchInfo> matches = new ArrayList<>();
1641
1642         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
1643
1644         if (destPrefix instanceof Inet4Address) {
1645             matches.add(MatchEthernetType.IPV4);
1646             if (prefixLength != 0) {
1647                 matches.add(new MatchIpv4Destination(destPrefix.getHostAddress(), Integer.toString(prefixLength)));
1648             }
1649         } else {
1650             matches.add(MatchEthernetType.IPV6);
1651             if (prefixLength != 0) {
1652                 matches.add(new MatchIpv6Destination(destPrefix.getHostAddress() + "/" + prefixLength));
1653             }
1654         }
1655
1656         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1657         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1658         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority,
1659             flowRef, 0, 0,
1660             COOKIE_VM_FIB_TABLE, matches, instructions);
1661
1662         Flow flow = flowEntity.getFlowBuilder().build();
1663         String flowId = flowEntity.getFlowId();
1664         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1665         Node nodeDpn = buildDpnNode(dpId);
1666
1667         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1668             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1669             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1670
1671         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1672             SubTransaction subTransaction = new SubTransactionImpl();
1673             if (addOrRemove == NwConstants.ADD_FLOW) {
1674                 subTransaction.setInstanceIdentifier(flowInstanceId);
1675                 subTransaction.setInstance(flow);
1676                 subTransaction.setAction(SubTransaction.CREATE);
1677             } else {
1678                 subTransaction.setInstanceIdentifier(flowInstanceId);
1679                 subTransaction.setAction(SubTransaction.DELETE);
1680             }
1681             transactionObjects.add(subTransaction);
1682         }
1683
1684         if (addOrRemove == NwConstants.ADD_FLOW) {
1685             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
1686         } else {
1687             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1688         }
1689
1690         if (!wrTxPresent) {
1691             tx.submit();
1692         }
1693     }
1694
1695     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1696     private Node buildDpnNode(BigInteger dpnId) {
1697         NodeId nodeId = new NodeId("openflow:" + dpnId);
1698         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1699
1700         return nodeDpn;
1701     }
1702
1703     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1704                                     int addOrRemove, WriteTransaction tx) {
1705         Boolean wrTxPresent = true;
1706         if (tx == null) {
1707             wrTxPresent = false;
1708             tx = dataBroker.newWriteOnlyTransaction();
1709         }
1710
1711         List<MatchInfo> matches = new ArrayList<>();
1712         matches.add(MatchEthernetType.MPLS_UNICAST);
1713         matches.add(new MatchMplsLabel(label));
1714
1715         // Install the flow entry in L3_LFIB_TABLE
1716         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1717
1718         FlowEntity flowEntity;
1719         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1720             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1721         Flow flow = flowEntity.getFlowBuilder().build();
1722         String flowId = flowEntity.getFlowId();
1723         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1724         Node nodeDpn = buildDpnNode(dpId);
1725         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1726             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1727             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1728
1729         if (addOrRemove == NwConstants.ADD_FLOW) {
1730             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
1731         } else {
1732             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1733         }
1734         if (!wrTxPresent) {
1735             tx.submit();
1736         }
1737
1738         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1739             dpId, label, instructions, flowKey, (NwConstants.ADD_FLOW == addOrRemove) ? "ADDED" : "REMOVED");
1740     }
1741
1742     private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress,
1743                                       final String ipPrefixAddress) {
1744         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}", dpId, vpnId, ipAddress);
1745         try {
1746             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress);
1747         } catch (NullPointerException e) {
1748             LOG.trace("", e);
1749         }
1750     }
1751
1752     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1753                                     final FutureCallback<List<Void>> callback) {
1754         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1755         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1756         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1757         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1758         if (!vrfTable.isPresent()) {
1759             LOG.warn("VRF Table not yet available for RD {}", rd);
1760             if (callback != null) {
1761                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1762                 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1763                 Futures.addCallback(listenableFuture, callback);
1764             }
1765             return;
1766         }
1767         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1768         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1769             () -> {
1770                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1771                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1772                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1773                     for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1774                         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1775                         if (subnetRoute != null) {
1776                             long elanTag = subnetRoute.getElantag();
1777                             installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1778                             continue;
1779                         }
1780                         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1781                         if (routerInt != null) {
1782                             LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1783                                 rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1784                             installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
1785                                 routerInt.getIpAddress(),
1786                                 new MacAddress(routerInt.getMacAddress()), NwConstants.ADD_FLOW);
1787                             continue;
1788                         }
1789                         //Handle local flow creation for imports
1790                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1791                             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1792                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1793                                 && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1794                                 if (lri.getDpnId().equals(dpnId)) {
1795                                     createLocalFibEntry(vpnId, rd, vrfEntry);
1796                                     continue;
1797                                 }
1798                             }
1799                         }
1800                         // Passing null as we don't know the dpn
1801                         // to which prefix is attached at this point
1802                         createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1803                     }
1804                     //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1805                     futures.add(tx.submit());
1806                 }
1807                 if (callback != null) {
1808                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1809                     Futures.addCallback(listenableFuture, callback);
1810                 }
1811                 return futures;
1812             });
1813     }
1814
1815
1816     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1817                                             final String localNextHopIp, final String remoteNextHopIp) {
1818         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1819             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1820         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1821         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1822         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1823         if (vrfTable.isPresent()) {
1824             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1825             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1826                 () -> {
1827                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1828                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1829                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1830                         for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1831                             if (!vrfEntry.getNextHopAddressList().isEmpty()) {
1832                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1833                                     if (remoteNextHopIp.trim()
1834                                         .equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1835                                         LOG.trace(" creating remote FIB entry for prefix {} rd {}",
1836                                             vrfEntry.getDestPrefix(), rd);
1837                                         createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(),
1838                                             vrfEntry, writeCfgTxn);
1839                                     }
1840                                 }
1841                             }
1842                         }
1843                         futures.add(writeCfgTxn.submit());
1844                     }
1845                     return futures;
1846                 });
1847         }
1848     }
1849
1850     public void populateInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1851                                             final String localNextHopIp, final String remoteNextHopIp) {
1852         LOG.trace("populateInternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1853             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1854         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1855         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1856         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1857         if (vrfTable.isPresent()) {
1858             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1859             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1860                 () -> {
1861                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1862                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1863                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1864                         for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1865                             // Handle Internal Routes only (i.e., STATIC for now)
1866                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC) {
1867                                 SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1868                                     /* Ignore SubnetRoute entry */
1869                                 if (subnetRoute == null) {
1870                                     if (!vrfEntry.getNextHopAddressList().isEmpty()) {
1871                                         if (remoteNextHopIp.trim()
1872                                             .equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1873                                             LOG.trace(" creating remote FIB entry for prefix {} rd {} on Dpn {}",
1874                                                 vrfEntry.getDestPrefix(), rd, dpnId);
1875                                             createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry,
1876                                                 writeCfgTxn);
1877                                         }
1878                                     }
1879                                 }
1880                             }
1881                         }
1882                         futures.add(writeCfgTxn.submit());
1883                     }
1884                     return futures;
1885                 });
1886         }
1887     }
1888
1889     public void manageRemoteRouteOnDPN(final boolean action,
1890                                        final BigInteger localDpnId,
1891                                        final long vpnId,
1892                                        final String rd,
1893                                        final String destPrefix,
1894                                        final String destTepIp) {
1895         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1896
1897         if (vpnInstance == null) {
1898             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1899             return;
1900         }
1901         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1902         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + localDpnId.toString(),
1903             () -> {
1904                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1905                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1906                     WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1907                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1908                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1909                     if (vrfEntry == null) {
1910                         return futures;
1911                     }
1912                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1913                         action, localDpnId, vpnId, rd, destPrefix);
1914                     List<String> nextHopAddressList = vrfEntry.getNextHopAddressList();
1915                     VrfEntry modVrfEntry;
1916                     if (nextHopAddressList == null || (nextHopAddressList.isEmpty())) {
1917                         List<String> nhList = Arrays.asList(destTepIp);
1918                         modVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(nhList).build();
1919                     } else {
1920                         modVrfEntry = vrfEntry;
1921                     }
1922
1923                     if (action == true) {
1924                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1925                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
1926                     } else {
1927                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1928                         deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
1929                     }
1930                     futures.add(writeTransaction.submit());
1931                 }
1932                 return futures;
1933             });
1934     }
1935
1936     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1937                                  final FutureCallback<List<Void>> callback) {
1938         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1939         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1940         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1941         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1942         if (vrfTable.isPresent()) {
1943             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1944             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1945                 () -> {
1946                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1947                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1948                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1949                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1950                                 /* Handle subnet routes here */
1951                             SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1952                             if (subnetRoute != null) {
1953                                 LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn",
1954                                     vrfEntry.getDestPrefix(),
1955                                     dpnId, rd);
1956                                 makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1957                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY,
1958                                     NwConstants.DEL_FLOW, tx);
1959                                 LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}",
1960                                     vrfEntry.getLabel(), rd,
1961                                     vrfEntry.getDestPrefix());
1962                                 continue;
1963                             }
1964                             // ping responder for router interfaces
1965                             RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1966                             if (routerInt != null) {
1967                                 LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1968                                     rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1969                                 installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
1970                                     routerInt.getIpAddress(),
1971                                     new MacAddress(routerInt.getMacAddress()), NwConstants.DEL_FLOW);
1972                                 continue;
1973                             }
1974                             // Passing null as we don't know the dpn
1975                             // to which prefix is attached at this point
1976                             deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1977                         }
1978                         futures.add(tx.submit());
1979                         if (callback != null) {
1980                             ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1981                             Futures.addCallback(listenableFuture, callback);
1982                         }
1983                     }
1984                     return futures;
1985                 });
1986         }
1987     }
1988
1989     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1990                                            final String localNextHopIp, final String remoteNextHopIp) {
1991         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1992                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1993             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1994         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1995         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1996         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1997         if (vrfTable.isPresent()) {
1998             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1999             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2000                 () -> {
2001                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2002                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2003                         WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
2004                         for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
2005                             if (!vrfEntry.getNextHopAddressList().isEmpty()) {
2006                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2007                                     if (remoteNextHopIp.trim()
2008                                         .equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
2009                                         LOG.trace(" deleting remote FIB entry {}", vrfEntry);
2010                                         deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(),
2011                                             vrfEntry, writeTransaction);
2012                                     }
2013                                 }
2014                             }
2015                         }
2016                         futures.add(writeTransaction.submit());
2017                     }
2018                     return futures;
2019                 });
2020         }
2021     }
2022
2023     public void cleanUpInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
2024                                            final String localNextHopIp, final String remoteNextHopIp) {
2025         LOG.trace("cleanUpInternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
2026                 + " localNexthopIp {} , remoteNexhtHopIp {}",
2027             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2028         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2029         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2030         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2031         if (vrfTable.isPresent()) {
2032             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2033             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2034                 () -> {
2035                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2036                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2037                         WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
2038                         for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
2039                             // Handle Internal Routes only (i.e, STATIC for now)
2040                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC) {
2041                                 SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
2042                                     /* Ignore SubnetRoute entry */
2043                                 if (subnetRoute == null) {
2044                                     if (!vrfEntry.getNextHopAddressList().isEmpty()) {
2045                                         if (remoteNextHopIp.trim()
2046                                             .equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
2047                                             LOG.trace(" deleting remote FIB entry {}", vrfEntry);
2048                                             deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(),
2049                                                 vrfEntry, writeTransaction);
2050                                         }
2051                                     }
2052                                 }
2053                             }
2054                         }
2055                         futures.add(writeTransaction.submit());
2056                     }
2057                     return futures;
2058                 });
2059         }
2060     }
2061
2062     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
2063         InstanceIdentifierBuilder<VrfTables> idBuilder =
2064             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
2065         InstanceIdentifier<VrfTables> id = idBuilder.build();
2066         return id;
2067     }
2068
2069     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
2070         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2071             .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(label).append(NwConstants.FLOWID_SEPARATOR)
2072             .append(priority).toString();
2073     }
2074
2075     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
2076         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2077             .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
2078             .append(rd).append(NwConstants.FLOWID_SEPARATOR)
2079             .append(priority).append(NwConstants.FLOWID_SEPARATOR)
2080             .append(destPrefix.getHostAddress()).toString();
2081     }
2082
2083     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
2084         return new StringBuilder(64).append(FLOWID_PREFIX)
2085             .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
2086             .append(prefix).append(NwConstants.FLOWID_SEPARATOR)
2087             .append(nextHop).toString();
2088     }
2089
2090     protected List<AdjacencyResult> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
2091                                                      final VrfEntry vrfEntry, String rd) {
2092         List<AdjacencyResult> adjacencyList = new ArrayList<>();
2093         List<String> prefixIpList = new ArrayList<>();
2094         LOG.trace("resolveAdjacency called with remotedDpnId {}, vpnId{}, VrfEntry {}",
2095             remoteDpnId, vpnId, vrfEntry);
2096         try {
2097             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2098                 Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
2099                 if (extraRoute == null) {
2100                     prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
2101                 } else {
2102                     prefixIpList = new ArrayList<>();
2103                     for (String extraRouteIp : extraRoute.getNexthopIpList()) {
2104                         prefixIpList.add(extraRouteIp + "/32");
2105                     }
2106                 }
2107             } else {
2108                 prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
2109             }
2110
2111             for (String prefixIp : prefixIpList) {
2112                 for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
2113                     LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
2114                     AdjacencyResult adjacencyResult = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
2115                         prefixIp, nextHopIp);
2116                     if (adjacencyResult != null && !adjacencyList.contains(adjacencyResult)) {
2117                         adjacencyList.add(adjacencyResult);
2118                     }
2119                 }
2120             }
2121         } catch (NullPointerException e) {
2122             LOG.trace("", e);
2123         }
2124         return adjacencyList;
2125     }
2126
2127     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
2128         InstanceIdentifier<VpnInstanceOpDataEntry> id =
2129             InstanceIdentifier.create(VpnInstanceOpData.class)
2130                 .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
2131         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
2132             FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
2133         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
2134     }
2135
2136     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2137         return new StringBuffer().append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2138             .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(tableMiss)
2139             .append(FLOWID_PREFIX).toString();
2140     }
2141
2142     /*
2143      * Install flow entry in protocol table to forward mpls
2144      * coming through gre tunnel to LFIB table.
2145      */
2146     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
2147         final BigInteger cookieProtocolTable = new BigInteger("1070000", 16);
2148         // Instruction to goto L3 InterfaceTable
2149         List<InstructionInfo> instructions = new ArrayList<>();
2150         instructions.add(new InstructionGotoTable(NwConstants.L3_LFIB_TABLE));
2151         List<MatchInfo> matches = new ArrayList<>();
2152         matches.add(MatchEthernetType.MPLS_UNICAST);
2153         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2154             getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2155                 NwConstants.L3_LFIB_TABLE),
2156             DEFAULT_FIB_FLOW_PRIORITY,
2157             "Protocol Table For LFIB",
2158             0, 0,
2159             cookieProtocolTable,
2160             matches, instructions);
2161
2162         if (addOrRemove == NwConstants.ADD_FLOW) {
2163             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2164             mdsalManager.installFlow(flowEntityToLfib);
2165         } else {
2166             mdsalManager.removeFlow(flowEntityToLfib);
2167         }
2168     }
2169
2170     public List<String> printFibEntries() {
2171         List<String> result = new ArrayList<>();
2172         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2173         result.add("-------------------------------------------------------------------");
2174         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2175         Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2176         if (fibEntries.isPresent()) {
2177             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2178             for (VrfTables vrfTable : vrfTables) {
2179                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2180                     for (String nextHop : vrfEntry.getNextHopAddressList()) {
2181                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2182                             vrfTable.getRouteDistinguisher(),
2183                             vrfEntry.getDestPrefix(), nextHop, vrfEntry.getLabel(), vrfEntry.getOrigin()));
2184                     }
2185                     if (vrfEntry.getNextHopAddressList().isEmpty()) {
2186                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2187                             vrfTable.getRouteDistinguisher(),
2188                             vrfEntry.getDestPrefix(), "local", vrfEntry.getLabel(), vrfEntry.getOrigin()));
2189                     }
2190                 }
2191             }
2192         }
2193         return result;
2194     }
2195
2196
2197     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2198         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2199             .child(VrfTables.class, new VrfTablesKey(rd))
2200             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2201         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2202         if (vrfEntry.isPresent()) {
2203             return vrfEntry.get();
2204         }
2205         return null;
2206     }
2207
2208     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2209         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2210             .child(VrfTables.class, new VrfTablesKey(rd))
2211             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2212         return vrfEntryId;
2213     }
2214
2215     protected Boolean isIpv4Address(String ipAddress) {
2216         try {
2217             InetAddress address = InetAddress.getByName(ipAddress);
2218             if (address instanceof Inet4Address) {
2219                 return true;
2220             }
2221         } catch (UnknownHostException e) {
2222             LOG.warn("Invalid ip address {}", ipAddress, e);
2223             return false;
2224         }
2225         return false;
2226     }
2227
2228     protected Boolean installRouterFibEntries(final VrfEntry vrfEntry, final Collection<VpnToDpnList> vpnToDpnList,
2229                                               long vpnId, int addOrRemove) {
2230         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
2231         if (routerInt != null && vpnToDpnList != null) {
2232             String routerId = routerInt.getUuid();
2233             String macAddress = routerInt.getMacAddress();
2234             String ipValue = routerInt.getIpAddress();
2235             LOG.trace("createFibEntries - Router augmented vrfentry found for for router uuid:{}, ip:{}, mac:{}",
2236                 routerId, ipValue, macAddress);
2237             for (VpnToDpnList vpnDpn : vpnToDpnList) {
2238                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
2239                     installRouterFibEntry(vrfEntry, vpnDpn.getDpnId(), vpnId, routerId, ipValue,
2240                         new MacAddress(macAddress), addOrRemove);
2241                 }
2242             }
2243             return true;
2244         }
2245         return false;
2246     }
2247
2248     public void installRouterFibEntry(final VrfEntry vrfEntry, BigInteger dpnId, long vpnId, String routerUuid,
2249                                       String routerInternalIp, MacAddress routerMac, int addOrRemove) {
2250         String[] subSplit = routerInternalIp.split("/");
2251         if (!isIpv4Address(subSplit[0])) {
2252             // Ping responder using OpenFlow rules is only supported for IPv4, hence skipping.
2253             return;
2254         }
2255
2256         String addRemoveStr = (addOrRemove == NwConstants.ADD_FLOW) ? "ADD_FLOW" : "DELETE_FLOW";
2257         LOG.trace("{}: bulding Echo Flow entity for dpid:{}, router_ip:{}, vpnId:{}, subSplit:{} ", addRemoveStr,
2258             dpnId, routerInternalIp, vpnId, subSplit[0]);
2259
2260         List<MatchInfo> matches = new ArrayList<>();
2261
2262         matches.add(MatchIpProtocol.ICMP);
2263         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
2264         matches.add(new MatchIcmpv4((short) 8, (short) 0));
2265         matches.add(MatchEthernetType.IPV4);
2266         matches.add(new MatchIpv4Destination(subSplit[0], "32"));
2267
2268         List<ActionInfo> actionsInfos = new ArrayList<>();
2269
2270         // Set Eth Src and Eth Dst
2271         actionsInfos.add(new ActionMoveSourceDestinationEth());
2272         actionsInfos.add(new ActionSetFieldEthernetSource(routerMac));
2273
2274         // Move Ip Src to Ip Dst
2275         actionsInfos.add(new ActionMoveSourceDestinationIp());
2276         actionsInfos.add(new ActionSetSourceIp(subSplit[0], "32"));
2277
2278         // Set the ICMP type to 0 (echo reply)
2279         actionsInfos.add(new ActionSetIcmpType((short) 0));
2280
2281         actionsInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
2282
2283         actionsInfos.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2284
2285         List<InstructionInfo> instructions = new ArrayList<>();
2286
2287         instructions.add(new InstructionApplyActions(actionsInfos));
2288
2289         int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH;
2290         String flowRef = getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, vrfEntry.getLabel(), priority);
2291
2292         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef,
2293             0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions);
2294
2295         if (addOrRemove == NwConstants.ADD_FLOW) {
2296             mdsalManager.installFlow(flowEntity);
2297         } else {
2298             mdsalManager.removeFlow(flowEntity);
2299         }
2300     }
2301
2302     public void removeInterVPNLinkRouteFlows(final String interVpnLinkName,
2303                                              final boolean isVpnFirstEndPoint,
2304                                              final VrfEntry vrfEntry) {
2305         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
2306             && vrfEntry.getNextHopAddressList().size() == 1);
2307         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLinkName);
2308
2309         if (!interVpnLinkState.isPresent()) {
2310             LOG.warn("Could not find State for InterVpnLink {}", interVpnLinkName);
2311             return;
2312         }
2313
2314         List<BigInteger> targetDpns =
2315             isVpnFirstEndPoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
2316                 : interVpnLinkState.get().getSecondEndpointState().getDpId();
2317
2318         String nextHop = vrfEntry.getNextHopAddressList().get(0);
2319
2320         // delete from FIB
2321         //
2322         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2323         FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2324         Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef)).setTableId(NwConstants.L3_FIB_TABLE)
2325             .setFlowName(flowRef).build();
2326
2327         LOG.trace("Removing flow in FIB table for interVpnLink {} key {}",
2328             interVpnLinkName, flowRef);
2329
2330         for (BigInteger dpId : targetDpns) {
2331             LOG.debug("Removing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2332                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), nextHop,
2333                 dpId, interVpnLinkName);
2334
2335             mdsalManager.removeFlow(dpId, flow);
2336         }
2337
2338         // delete from LFIB
2339         //
2340         LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2341
2342         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
2343         for (BigInteger dpId : targetDpns) {
2344             LOG.debug("Removing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
2345                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), nextHop,
2346                 dpId, interVpnLinkName);
2347             makeLFibTableEntry(dpId, vrfEntry.getLabel(), null /* no instructions */,
2348                 LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, tx);
2349         }
2350         tx.submit();
2351     }
2352 }