vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.util.concurrent.FutureCallback;
  13 import com.google.common.util.concurrent.Futures;
  14 import com.google.common.util.concurrent.ListenableFuture;
  15 import java.math.BigInteger;
  16 import java.net.Inet4Address;
  17 import java.net.InetAddress;
  18 import java.net.UnknownHostException;
  19 import java.util.ArrayList;
  20 import java.util.Arrays;
  21 import java.util.Collection;
  22 import java.util.Collections;
  23 import java.util.List;
  24 import java.util.concurrent.BlockingQueue;
  25 import java.util.concurrent.Callable;
  26 import java.util.concurrent.ExecutionException;
  27 import java.util.concurrent.Future;
  28 import java.util.concurrent.LinkedBlockingQueue;
  29 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  30 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  31 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  32 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  33 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  34 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  35 import org.opendaylight.genius.mdsalutil.ActionInfo;
  36 import org.opendaylight.genius.mdsalutil.FlowEntity;
  37 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  38 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  39 import org.opendaylight.genius.mdsalutil.MatchInfo;
  40 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  41 import org.opendaylight.genius.mdsalutil.NwConstants;
  42 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  43 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationEth;
  44 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationIp;
  45 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionSetIcmpType;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionSetSourceIp;
  55 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  56 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  57 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  58 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  59 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  60 import org.opendaylight.genius.mdsalutil.matches.MatchIcmpv4;
  61 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  62 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  63 import org.opendaylight.genius.mdsalutil.matches.MatchIpv6Destination;
  64 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  65 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  67 import org.opendaylight.genius.utils.ServiceIndex;
  68 import org.opendaylight.genius.utils.batching.ActionableResource;
  69 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  70 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  71 import org.opendaylight.genius.utils.batching.ResourceHandler;
  72 import org.opendaylight.genius.utils.batching.SubTransaction;
  73 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
  74 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  75 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  76 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  77 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  78 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  79 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  80 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.Tunnel;
  81 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeInputBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeOutput;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroute;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.Vpn;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.VpnKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 135 import org.opendaylight.yangtools.yang.binding.DataObject;
 136 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 137 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 138 import org.opendaylight.yangtools.yang.common.RpcResult;
 139 import org.slf4j.Logger;
 140 import org.slf4j.LoggerFactory;
 141
 142 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener>
 143     implements AutoCloseable, ResourceHandler {
 144
 145     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 146     private static final String FLOWID_PREFIX = "L3.";
 147     private final DataBroker dataBroker;
 148     private final IMdsalApiManager mdsalManager;
 149     private IVpnManager vpnmanager;
 150     private final NexthopManager nextHopManager;
 151     private ItmRpcService itmManager;
 152     private final OdlInterfaceRpcService interfaceManager;
 153     private final IdManagerService idManager;
 154     private static final BigInteger COOKIE_VM_FIB_TABLE = new BigInteger("8000003", 16);
 155     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 156     private static final int LFIB_INTERVPN_PRIORITY = 1;
 157     private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
 158     private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
 159     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 160     List<SubTransaction> transactionObjects;
 161     private static final int PERIODICITY = 500;
 162     private static Integer batchSize;
 163     private static Integer batchInterval;
 164     private static final int BATCH_SIZE = 1000;
 165     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 166     private final ResourceBatchingManager resourceBatchingManager;
 167
 168     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 169                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
 170                             final IdManagerService idManager) {
 171         super(VrfEntry.class, VrfEntryListener.class);
 172         this.dataBroker = dataBroker;
 173         this.mdsalManager = mdsalApiManager;
 174         this.nextHopManager = nexthopManager;
 175         this.interfaceManager = interfaceManager;
 176         this.idManager = idManager;
 177
 178         batchSize = Integer.getInteger("batch.size");
 179         if (batchSize == null) {
 180             batchSize = BATCH_SIZE;
 181         }
 182         batchInterval = Integer.getInteger("batch.wait.time");
 183         if (batchInterval == null) {
 184             batchInterval = PERIODICITY;
 185         }
 186         resourceBatchingManager = ResourceBatchingManager.getInstance();
 187         resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
 188         transactionObjects = new ArrayList<>();
 189     }
 190
 191     public void start() {
 192         LOG.info("{} start", getClass().getSimpleName());
 193         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 194     }
 195
 196     @Override
 197     protected VrfEntryListener getDataTreeChangeListener() {
 198         return VrfEntryListener.this;
 199     }
 200
 201     @Override
 202     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 203         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 204     }
 205
 206     @Override
 207     public DataBroker getResourceBroker() {
 208         return dataBroker;
 209     }
 210
 211     @Override
 212     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 213         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 214         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 215         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} nexthop {} label {}",
 216             rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 217         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 218             createFibEntries(identifier, vrfEntry);
 219         } else {
 220             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 221             actResource.setAction(ActionableResource.CREATE);
 222             actResource.setInstanceIdentifier(identifier);
 223             actResource.setInstance(vrfEntry);
 224             vrfEntryBufferQ.add(actResource);
 225         }
 226         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 227         LOG.info("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
 228             rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 229     }
 230
 231     @Override
 232     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 233         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 234         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 235         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} nexthop {} label {}",
 236             rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 237         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 238             deleteFibEntries(identifier, vrfEntry);
 239         } else {
 240             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 241             actResource.setAction(ActionableResource.DELETE);
 242             actResource.setInstanceIdentifier(identifier);
 243             actResource.setInstance(vrfEntry);
 244             vrfEntryBufferQ.add(actResource);
 245         }
 246         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 247         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
 248             rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 249     }
 250
 251     @Override
 252     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 253         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 254
 255         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 256         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
 257         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
 258             rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 259         // Handle BGP Routes first
 260         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 261             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + update.getDestPrefix());
 262             actResource.setAction(ActionableResource.UPDATE);
 263             actResource.setInstanceIdentifier(identifier);
 264             actResource.setInstance(update);
 265             actResource.setOldInstance(original);
 266             vrfEntryBufferQ.add(actResource);
 267             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 268                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 269             return;
 270         }
 271
 272         // Handle Internal Routes next (ie., STATIC only)
 273         if (FibUtil.isControllerManagedNonInterVpnLinkRoute(RouteOrigin.value(update.getOrigin()))) {
 274             SubnetRoute subnetRoute = update.getAugmentation(SubnetRoute.class);
 275             /* Ignore SubnetRoute entry, as it will be driven by createFibEntries call down below */
 276             if (subnetRoute == null) {
 277                 List<String> origNhList = original.getNextHopAddressList();
 278                 List<String> updateNhList = update.getNextHopAddressList();
 279                 //final SubnetRoute subnetRoute = update.getAugmentation(SubnetRoute.class);
 280                 LOG.info("UPDATE: Original nexthop {} updateNextHop {} ", origNhList, updateNhList);
 281
 282                 // If original VRF Entry had nexthop null , but update VRF Entry
 283                 // has nexthop , route needs to be created on remote Dpns
 284                 if (((origNhList == null) || (origNhList.isEmpty())
 285                     && (updateNhList != null) && (!updateNhList.isEmpty()))) {
 286                     // TODO(vivek): Though ugly, Not handling this code now, as each
 287                     // tep add event will invoke flow addition
 288                     LOG.trace("Original VRF entry NH is null for destprefix {}. This event is IGNORED here.",
 289                         update.getDestPrefix());
 290                     return;
 291                 }
 292
 293                 // If original VRF Entry had valid nexthop , but update VRF Entry
 294                 // has nexthop empty'ed out, route needs to be removed from remote Dpns
 295                 if (((updateNhList == null) || (updateNhList.isEmpty())
 296                     && (origNhList != null) && (!origNhList.isEmpty()))) {
 297                     LOG.trace("Original VRF entry had valid NH for destprefix {}. This event is IGNORED here.",
 298                         update.getDestPrefix());
 299                     return;
 300                 }
 301             }
 302             createFibEntries(identifier, update);
 303             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 304                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 305             return;
 306         }
 307
 308         /* Handl all other route origins */
 309         createFibEntries(identifier, update);
 310
 311         LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 312             rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 313     }
 314
 315     @Override
 316     public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 317                        Object original, Object update, List<SubTransaction> transactionObjects) {
 318         this.transactionObjects = transactionObjects;
 319         if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 320             createFibEntries(tx, identifier, (VrfEntry) update);
 321         }
 322     }
 323
 324     @Override
 325     public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 326                        Object vrfEntry, List<SubTransaction> transactionObjects) {
 327         this.transactionObjects = transactionObjects;
 328         if (vrfEntry instanceof VrfEntry) {
 329             createFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 330         }
 331     }
 332
 333     @Override
 334     public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 335                        Object vrfEntry, List<SubTransaction> transactionObjects) {
 336         this.transactionObjects = transactionObjects;
 337         if (vrfEntry instanceof VrfEntry) {
 338             deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 339         }
 340     }
 341
 342     @Override
 343     public int getBatchSize() {
 344         return batchSize;
 345     }
 346
 347     @Override
 348     public int getBatchInterval() {
 349         return batchInterval;
 350     }
 351
 352     @Override
 353     public LogicalDatastoreType getDatastoreType() {
 354         return LogicalDatastoreType.CONFIGURATION;
 355     }
 356
 357     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 358         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 359
 360         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 361         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 362         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 363             + " has null vpnId!");
 364
 365         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 366         final Long vpnId = vpnInstance.getVpnId();
 367         final String rd = vrfTableKey.getRouteDistinguisher();
 368         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 369         if (subnetRoute != null) {
 370             final long elanTag = subnetRoute.getElantag();
 371             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 372                 rd, vrfEntry.getDestPrefix(), elanTag);
 373             if (vpnToDpnList != null) {
 374                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 375                 dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
 376                     () -> {
 377                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 378                         for (final VpnToDpnList curDpn : vpnToDpnList) {
 379                             if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 380                                 installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 381                             }
 382                         }
 383                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 384                         futures.add(tx.submit());
 385                         return futures;
 386                     });
 387             }
 388             return;
 389         }
 390         // ping responder for router interfaces
 391         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnId, NwConstants.ADD_FLOW)) {
 392             return;
 393         }
 394
 395         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 396
 397         if (vpnToDpnList != null) {
 398             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 399             dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
 400                 () -> {
 401                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 402                     for (VpnToDpnList vpnDpn : vpnToDpnList) {
 403                         if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 404                             if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 405                                 createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 406                                     vrfTableKey, vrfEntry, tx);
 407                             }
 408                         }
 409                     }
 410                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 411                     futures.add(tx.submit());
 412                     return futures;
 413                 });
 414         }
 415
 416         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 417         if (optVpnUuid.isPresent()) {
 418             Optional<InterVpnLinkDataComposite> optInterVpnLink =
 419                 InterVpnLinkCache.getInterVpnLinkByVpnId(optVpnUuid.get());
 420             LOG.debug("InterVpnLink {} found in Cache: {}", optVpnUuid.get(), optInterVpnLink.isPresent());
 421             if (optInterVpnLink.isPresent()) {
 422                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
 423                 String vpnUuid = optVpnUuid.get();
 424                 String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
 425                 if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 426                     // This is an static route that points to the other endpoint of an InterVpnLink
 427                     // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 428                     installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 429                     installInterVpnRouteInLFib(rd, vrfEntry);
 430                 }
 431             }
 432         }
 433     }
 434
 435
 436     /*
 437       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 438       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 439       provided by ResourceBatchingManager
 440      */
 441     private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
 442                                   final VrfEntry vrfEntry) {
 443         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 444
 445         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 446         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 447         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 448             + " has null vpnId!");
 449
 450         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 451         final String rd = vrfTableKey.getRouteDistinguisher();
 452         if (vpnToDpnList != null) {
 453             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 454                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 455                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 456                 }
 457             }
 458         }
 459     }
 460
 461     private boolean originMustBeLeaked(InterVpnLink ivpnLink, RouteOrigin routeOrigin) {
 462         return routeOrigin == RouteOrigin.BGP && ivpnLink.isBgpRoutesLeaking()
 463             || routeOrigin == RouteOrigin.STATIC && ivpnLink.isStaticRoutesLeaking()
 464             || routeOrigin == RouteOrigin.CONNECTED && ivpnLink.isConnectedRoutesLeaking();
 465     }
 466
 467     // FIXME: Refactoring needed here.
 468     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 469     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 470                                    int addOrRemove) {
 471         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 472         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 473
 474         String rd = vrfTableKey.getRouteDistinguisher();
 475         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 476         if (vpnInstance == null) {
 477             LOG.error("VPN Instance not available for route with prefix {} label {} nextHop {} RD {}. Returning...",
 478                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
 479             return;
 480         }
 481         String vpnUuid = vpnInstance.getVpnInstanceName();
 482         Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(),
 483             "Could not find suitable VPN UUID for Route-Distinguisher=" + rd);
 484
 485         // if the new vrfEntry has been learned by Quagga BGP, its necessary to check if it's
 486         // there an interVpnLink for the involved vpn in order to make learn the new route to
 487         // the other part of the inter-vpn-link.
 488
 489         // For leaking, we need the InterVpnLink to be active. For removal, we just need a InterVpnLink.
 490         Optional<InterVpnLink> interVpnLink =
 491             (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(dataBroker, rd)
 492                 : FibUtil.getInterVpnLinkByRd(dataBroker, rd);
 493         if (!interVpnLink.isPresent()) {
 494             LOG.debug("Could not find an InterVpnLink for Route-Distinguisher={}", rd);
 495             return;
 496         }
 497
 498         // Ok, at this point everything is ready for the leaking/removal... but should it be performed?
 499         // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled.
 500         boolean proceed =
 501             addOrRemove == NwConstants.DEL_FLOW || originMustBeLeaked(interVpnLink.get(),
 502                 RouteOrigin.value(vrfEntry.getOrigin()));
 503
 504         if (proceed) {
 505             boolean isVpnFirstEndpoint = interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid);
 506
 507             String theOtherVpnId = isVpnFirstEndpoint ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
 508                 : interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue();
 509             String dstVpnRd = FibUtil.getVpnRd(dataBroker, theOtherVpnId);
 510             String endpointIp = isVpnFirstEndpoint ? interVpnLink.get().getFirstEndpoint().getIpAddress().getValue()
 511                 : interVpnLink.get().getSecondEndpoint().getIpAddress().getValue();
 512
 513             InstanceIdentifier<VrfEntry> vrfEntryIidInOtherVpn =
 514                 InstanceIdentifier.builder(FibEntries.class)
 515                     .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
 516                     .child(VrfEntry.class, new VrfEntryKey(vrfEntry.getDestPrefix()))
 517                     .build();
 518             if (addOrRemove == NwConstants.ADD_FLOW) {
 519                 LOG.debug("Leaking route (destination={}, nexthop={}) from Vrf={} to Vrf={}",
 520                     vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), rd, dstVpnRd);
 521                 String key = rd + FibConstants.SEPARATOR + vrfEntry.getDestPrefix();
 522                 long label = FibUtil.getUniqueId(idManager, FibConstants.VPN_IDPOOL_NAME, key);
 523                 VrfEntry newVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(Arrays.asList(endpointIp))
 524                     .setLabel(label)
 525                     .setOrigin(RouteOrigin.INTERVPN.getValue())
 526                     .build();
 527                 MDSALUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
 528                     vrfEntryIidInOtherVpn, newVrfEntry);
 529             } else {
 530                 LOG.debug("Removing leaked vrfEntry={}", vrfEntryIidInOtherVpn.toString());
 531                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn);
 532             }
 533         }
 534     }
 535
 536     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 537         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 538         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 539         prefixBuilder.setDpnId(lri.getDpnId());
 540         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 541         prefixBuilder.setIpAddress(lri.getPrefix());
 542         // Increment the refCount here
 543         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 544             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 545         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 546         if (!isPresentInList) {
 547             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 548             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 549             vpnInstanceNames.add(vpnInstanceName);
 550             builder.setVpnInstanceList(vpnInstanceNames);
 551             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(),
 552                 FibUtil.DEFAULT_CALLBACK);
 553         } else {
 554             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 555         }
 556         return prefixBuilder.build();
 557     }
 558
 559     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 560                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 561         Boolean wrTxPresent = true;
 562         if (tx == null) {
 563             wrTxPresent = false;
 564             tx = dataBroker.newWriteOnlyTransaction();
 565         }
 566         synchronized (vrfEntry.getLabel().toString().intern()) {
 567             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 568             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
 569                 && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 570
 571                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 572                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 573                         FibUtil.getVpnInstanceOpData(dataBroker, rd);
 574                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
 575                         String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 576                         if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 577                             updateVpnReferencesInLri(lri, vpnInstanceName, false);
 578                         }
 579                     }
 580                 }
 581                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 582                     vrfEntry.getLabel(), lri.getVpnInterfaceName(), lri.getDpnId());
 583             }
 584         }
 585         final List<InstructionInfo> instructions = new ArrayList<>();
 586         BigInteger subnetRouteMeta = ((BigInteger.valueOf(elanTag)).shiftLeft(32))
 587             .or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 588         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 589         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 590         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 591
 592         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 593             List<ActionInfo> actionsInfos = new ArrayList<>();
 594             // reinitialize instructions list for LFIB Table
 595             final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 596
 597             actionsInfos.add(new ActionPopMpls());
 598             LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 599             LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 600                 MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 601             LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 602
 603             makeLFibTableEntry(dpnId, vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 604                 NwConstants.ADD_FLOW, tx);
 605         }
 606         if (!wrTxPresent) {
 607             tx.submit();
 608         }
 609     }
 610
 611     /*
 612      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 613      * LportDispatcher table (via table 80)
 614      */
 615     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 616         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 617         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 618         // packet is commuted from Vpn2 to Vpn1.
 619         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
 620         if (!vpnNameOpc.isPresent()) {
 621             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 622             return;
 623         }
 624
 625         String vpnName = vpnNameOpc.get();
 626         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
 627         boolean interVpnLinkFound = false;
 628         for (InterVpnLink interVpnLink : interVpnLinks) {
 629             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 630             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 631                 && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 632             if (vpnIs1stEndpoint || vpnIs2ndEndpoint) {
 633                 interVpnLinkFound = true;
 634
 635                 Optional<InterVpnLinkState> vpnLinkState =
 636                     FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 637                 if (!vpnLinkState.isPresent()
 638                     || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active)) {
 639                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 640                         interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 641                         interVpnLink.getSecondEndpoint().getVpnUuid().getValue());
 642                     return;
 643                 }
 644
 645                 List<BigInteger> targetDpns = vpnIs1stEndpoint ? vpnLinkState.get().getFirstEndpointState().getDpId()
 646                     : vpnLinkState.get().getSecondEndpointState().getDpId();
 647                 Long lportTag = vpnIs1stEndpoint ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 648                     : vpnLinkState.get().getFirstEndpointState().getLportTag();
 649
 650                 LOG.trace("Installing flow in LFIB table for interVpnLink {}", interVpnLink.getName());
 651
 652                 for (BigInteger dpId : targetDpns) {
 653                     List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls());
 654
 655                     List<InstructionInfo> instructions =
 656                         Arrays.asList(new InstructionApplyActions(actionsInfos),
 657                             new InstructionWriteMetadata(
 658                                 MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 659                                     ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 660                                         NwConstants.L3VPN_SERVICE_INDEX)),
 661                                 MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 662                             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 663
 664                     LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for "
 665                         + "InterVpnLink {} in LFIB",
 666                         vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(),
 667                         dpId, interVpnLink.getName());
 668
 669                     makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY,
 670                         NwConstants.ADD_FLOW, null);
 671                 }
 672
 673                 break;
 674             }
 675         }
 676
 677         if (!interVpnLinkFound) {
 678             LOG.warn("VrfEntry=[prefix={} label={} nexthop={}] for VPN {} has origin INTERVPN but "
 679                 + "no InterVpnLink could be found",
 680                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
 681         }
 682     }
 683
 684
 685     /*
 686      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 687      */
 688     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 689                                                final VrfEntry vrfEntry, long vpnTag) {
 690         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 691         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 692             && vrfEntry.getNextHopAddressList().size() == 1);
 693         String destination = vrfEntry.getDestPrefix();
 694         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 695         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 696
 697         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 698         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 699         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 700         if (interVpnLink.getState().or(State.Error) != State.Active) {
 701             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 702                 destination, nextHop, interVpnLinkName);
 703             return;
 704         }
 705
 706         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 707         if (!optOtherEndpointLportTag.isPresent()) {
 708             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 709                 vpnUuid, interVpnLinkName);
 710             return;
 711         }
 712
 713         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 714         if (targetDpns.isEmpty()) {
 715             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 716                 vpnUuid, interVpnLinkName);
 717             return;
 718         }
 719
 720         String[] values = destination.split("/");
 721         String destPrefixIpAddress = values[0];
 722         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 723
 724         List<MatchInfo> matches = new ArrayList<>();
 725         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 726         matches.add(MatchEthernetType.IPV4);
 727
 728         if (prefixLength != 0) {
 729             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 730         }
 731
 732         List<Instruction> instructions =
 733             Arrays.asList(new InstructionWriteMetadata(
 734                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 735                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 736                             .L3VPN_SERVICE_INDEX)),
 737                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 738                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 739
 740         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 741         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 742         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 743             COOKIE_VM_FIB_TABLE, matches, instructions);
 744
 745         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 746             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 747
 748         for (BigInteger dpId : targetDpns) {
 749
 750             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nextHop={}] dpn {} for InterVpnLink {} in FIB",
 751                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(),
 752                 dpId, interVpnLink.getInterVpnLinkName());
 753
 754             mdsalManager.installFlow(dpId, flowEntity);
 755         }
 756     }
 757
 758
 759     // TODO Clean up the exception handling
 760     @SuppressWarnings("checkstyle:IllegalCatch")
 761     private <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 762                                                     InstanceIdentifier<T> path) {
 763
 764         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 765
 766         Optional<T> result = Optional.absent();
 767         try {
 768             result = tx.read(datastoreType, path).get();
 769         } catch (Exception e) {
 770             throw new RuntimeException(e);
 771         }
 772
 773         return result;
 774     }
 775
 776     private List<BigInteger> getDpnIdForPrefix(DataBroker broker, Long vpnId, String rd, VrfEntry vrfEntry) {
 777         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 778         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, vrfEntry.getDestPrefix());
 779
 780         if (localNextHopInfo == null) {
 781             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 782             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 783             if (extraRoute != null) {
 784                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 785                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 786                     if (nextHopIp != null) {
 787                         localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, nextHopIp + "/32");
 788                         if (localNextHopInfo != null) {
 789                             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 790                         }
 791                     }
 792                 }
 793             }
 794         } else {
 795             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 796         }
 797
 798         return returnLocalDpnId;
 799     }
 800
 801     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 802         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 803         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
 804         String localNextHopIP = vrfEntry.getDestPrefix();
 805
 806         if (localNextHopInfo == null) {
 807             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 808             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 809             if (extraRoute != null) {
 810                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 811                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 812                     if (nextHopIp != null) {
 813                         localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32");
 814                         localNextHopIP = nextHopIp + "/32";
 815                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 816                             rd, vrfEntry, vpnId);
 817                         returnLocalDpnId.add(dpnId);
 818                     }
 819                 }
 820             }
 821             if (localNextHopInfo == null) {
 822             /* imported routes case */
 823                 synchronized (vrfEntry.getLabel().toString().intern()) {
 824                     LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 825                     if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
 826                         && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 827                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 828                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 829                                 FibUtil.getVpnInstanceOpData(dataBroker, rd);
 830                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 831                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 832                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 833                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 834                                     localNextHopIP = lri.getPrefix();
 835                                 } else {
 836                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 837                                     localNextHopIP = lri.getPrefix();
 838                                 }
 839                             }
 840                             if (localNextHopInfo != null) {
 841                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 842                                     vrfEntry.getLabel(), localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 843                                 BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 844                                     vpnId, rd, vrfEntry, lri.getParentVpnid());
 845                                 returnLocalDpnId.add(dpnId);
 846                             }
 847                         }
 848                     }
 849                 }
 850             }
 851         } else {
 852             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 853             returnLocalDpnId.add(dpnId);
 854         }
 855
 856         return returnLocalDpnId;
 857     }
 858
 859     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 860                                                 final Long vpnId, final String rd,
 861                                                 final VrfEntry vrfEntry, Long parentVpnId) {
 862         if (localNextHopInfo != null) {
 863             final BigInteger dpnId = localNextHopInfo.getDpnId();
 864             if (!isVpnPresentInDpn(rd, dpnId)) {
 865                 LOG.error("The vpnName with vpnId {} rd {} is not available on dpn {}", vpnId, rd, dpnId.toString());
 866                 return BigInteger.ZERO;
 867             }
 868
 869             final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId,
 870                 localNextHopInfo.getVpnInterfaceName(), localNextHopIP, vrfEntry.getDestPrefix());
 871             if (groupId == 0) {
 872                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 873                     vrfEntry.getDestPrefix(), rd, localNextHopInfo.getVpnInterfaceName(), dpnId.toString());
 874                 return BigInteger.ZERO;
 875             }
 876             final List<InstructionInfo> instructions = Collections.singletonList(
 877                 new InstructionApplyActions(
 878                     Collections.singletonList(new ActionGroup(groupId))));
 879             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 880                 new InstructionApplyActions(
 881                     Arrays.asList(new ActionPopMpls(), new ActionGroup(groupId))));
 882             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 883                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 884                     dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
 885             } else {
 886                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. "
 887                     + "LFib and Terminating table entries will not be created.",
 888                     rd, vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpnId);
 889             }
 890             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 891             dataStoreCoordinator.enqueueJob("FIB-" + vpnId.toString()
 892                     + "-" + dpnId.toString() + "-" + vrfEntry.getDestPrefix(),
 893                 () -> {
 894                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 895                     makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 896                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 897                         makeLFibTableEntry(dpnId, vrfEntry.getLabel(), lfibinstructions,
 898                             DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 899                         makeTunnelTableEntry(dpnId, vrfEntry.getLabel(), groupId, tx);
 900                     }
 901                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 902                     futures.add(tx.submit());
 903                     return futures;
 904                 });
 905             return dpnId;
 906         }
 907         return BigInteger.ZERO;
 908     }
 909
 910     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId) {
 911         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 912         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 913         if (dpnInVpn.isPresent()) {
 914             return true;
 915         }
 916         return false;
 917     }
 918
 919     private LabelRouteInfo getLabelRouteInfo(Long label) {
 920         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 921             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 922         Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 923         if (opResult.isPresent()) {
 924             return opResult.get();
 925         }
 926         return null;
 927     }
 928
 929     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName) {
 930         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 931         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 932             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 933         if (lri == null) {
 934             return true;
 935         }
 936         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 937             ? lri.getVpnInstanceList() : new ArrayList<>();
 938         if (vpnInstancesList.contains(vpnInstanceName)) {
 939             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 940             vpnInstancesList.remove(vpnInstanceName);
 941         }
 942         if (vpnInstancesList.size() == 0) {
 943             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 944             FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 945             return true;
 946         } else {
 947             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 948             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 949             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(),
 950                 FibUtil.DEFAULT_CALLBACK);
 951         }
 952         return false;
 953     }
 954
 955     private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 956                                       WriteTransaction tx) {
 957         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 958
 959         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 960
 961         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 962             dpId, label, groupId);
 963     }
 964
 965     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 966                                                 WriteTransaction tx) {
 967         List<MatchInfo> mkMatches = new ArrayList<>();
 968
 969         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 970             destDpId, label, actionsInfos);
 971
 972         // Matching metadata
 973         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 974         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 975
 976         List<InstructionInfo> mkInstructions = new ArrayList<>();
 977         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 978
 979         FlowEntity terminatingServiceTableFlowEntity =
 980             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 981             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
 982                 String.format("%s:%d", "TST Flow Entry ", label),
 983             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
 984
 985         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
 986
 987         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 988
 989         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 990         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 991             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 992             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
 993             .child(Flow.class, flowKey).build();
 994         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(), true);
 995     }
 996
 997     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 998         FlowEntity flowEntity;
 999         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1000         List<MatchInfo> mkMatches = new ArrayList<>();
1001         // Matching metadata
1002         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1003         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1004             NwConstants.INTERNAL_TUNNEL_TABLE,
1005             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1006             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1007             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1008         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
1009         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1010         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1011             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1012             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1013
1014         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1015         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1016     }
1017
1018     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1019         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1020         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1021         String localNextHopIP = vrfEntry.getDestPrefix();
1022
1023         if (localNextHopInfo == null) {
1024             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1025             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1026             if (extraRoute != null) {
1027                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1028                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1029                     if (nextHopIp != null) {
1030                         localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
1031                         localNextHopIP = nextHopIp + "/32";
1032                         BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1033                             vpnId, rd, vrfEntry, true /*isExtraRoute*/);
1034                         if (!dpnId.equals(BigInteger.ZERO)) {
1035                             returnLocalDpnId.add(dpnId);
1036                         }
1037                     }
1038                 }
1039             }
1040
1041             if (localNextHopInfo == null) {
1042               /* Imported VRF entry */
1043                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1044                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1045                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1046                     VpnNexthopBuilder vpnNexthopBuilder = new VpnNexthopBuilder();
1047                     vpnNexthopBuilder.setDpnId(lri.getDpnId());
1048                     BigInteger dpnId = checkDeleteLocalFibEntry(vpnNexthopBuilder.build(), localNextHopIP,
1049                         vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1050                     if (!dpnId.equals(BigInteger.ZERO)) {
1051                         returnLocalDpnId.add(dpnId);
1052                     }
1053                 }
1054             }
1055
1056         } else {
1057             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1058                 vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1059             if (!dpnId.equals(BigInteger.ZERO)) {
1060                 returnLocalDpnId.add(dpnId);
1061             }
1062         }
1063
1064         return returnLocalDpnId;
1065     }
1066
1067     private BigInteger checkDeleteLocalFibEntry(VpnNexthop localNextHopInfo, final String localNextHopIP,
1068                                                 final Long vpnId, final String rd,
1069                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
1070         if (localNextHopInfo != null) {
1071             final BigInteger dpnId = localNextHopInfo.getDpnId();
1072             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1073             dataStoreCoordinator.enqueueJob("FIB-" + vpnId.toString() + "-"
1074                     + dpnId.toString() + "-" + vrfEntry.getDestPrefix(),
1075                 () -> {
1076                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1077                     makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
1078                         NwConstants.DEL_FLOW, tx);
1079                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1080                         makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null /* instructions */,
1081                             DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1082                         removeTunnelTableEntry(dpnId, vrfEntry.getLabel(), tx);
1083                     }
1084                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1085                     futures.add(tx.submit());
1086                     return futures;
1087                 });
1088             //TODO: verify below adjacency call need to be optimized (?)
1089             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1090             return dpnId;
1091         }
1092         return BigInteger.ZERO;
1093     }
1094
1095     private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
1096         return InstanceIdentifier.builder(VpnToExtraroute.class)
1097             .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
1098                 new ExtrarouteKey(ipPrefix)).build();
1099     }
1100
1101     private Extraroute getVpnToExtraroute(String rd, String ipPrefix) {
1102         Optional<Extraroute> extraRouteInfo =
1103             FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getVpnToExtrarouteIdentifier(rd, ipPrefix));
1104         return extraRouteInfo.isPresent() ? extraRouteInfo.get() : null;
1105
1106     }
1107
1108     private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
1109         try {
1110             Future<RpcResult<GetTunnelTypeOutput>> result = interfaceManager.getTunnelType(
1111                 new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
1112             RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
1113             if (!rpcResult.isSuccessful()) {
1114                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
1115             } else {
1116                 return rpcResult.getResult().getTunnelType();
1117             }
1118
1119         } catch (InterruptedException | ExecutionException e) {
1120             LOG.warn("Exception when getting tunnel interface Id for tunnel type", e);
1121         }
1122
1123         return null;
1124
1125     }
1126
1127     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1128                                       final VrfEntry vrfEntry, WriteTransaction tx) {
1129         Boolean wrTxPresent = true;
1130         if (tx == null) {
1131             wrTxPresent = false;
1132             tx = dataBroker.newWriteOnlyTransaction();
1133         }
1134         String rd = vrfTableKey.getRouteDistinguisher();
1135         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1136             vrfEntry.getDestPrefix(), rd, remoteDpnId);
1137
1138         List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1139         if (adjacencyResults.isEmpty()) {
1140             LOG.error("Could not get interface for nexthop: {} in vpn {}",
1141                 vrfEntry.getNextHopAddressList(), rd);
1142             LOG.warn("Failed to add Route: {} in vpn: {}",
1143                 vrfEntry.getDestPrefix(), rd);
1144             return;
1145         }
1146
1147         for (AdjacencyResult adjacencyResult : adjacencyResults) {
1148             List<ActionInfo> actionInfos = new ArrayList<>();
1149             String egressInterface = adjacencyResult.getInterfaceName();
1150             if (Tunnel.class.equals(adjacencyResult.getInterfaceType())) {
1151                 addTunnelInterfaceActions(egressInterface, vpnId, vrfEntry, actionInfos);
1152             } else {
1153                 addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1154             }
1155             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(egressInterface);
1156             if (egressActions.isEmpty()) {
1157                 LOG.error("Failed to retrieve egress action for prefix {} nextHop {} interface {}. "
1158                     + "Aborting remote FIB entry creation.",
1159                     vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), egressInterface);
1160                 return;
1161             }
1162             actionInfos.addAll(egressActions);
1163             List<InstructionInfo> instructions = new ArrayList<>();
1164             instructions.add(new InstructionApplyActions(actionInfos));
1165             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1166         }
1167         if (!wrTxPresent) {
1168             tx.submit();
1169         }
1170         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1171     }
1172
1173     private void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, List<ActionInfo> actionInfos) {
1174         String ipPrefix = vrfEntry.getDestPrefix();
1175         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, ipPrefix);
1176         if (prefixInfo == null) {
1177             LOG.debug("No prefix info found for prefix {}", ipPrefix);
1178             return;
1179         }
1180
1181         String ifName = prefixInfo.getVpnInterfaceName();
1182         if (ifName == null) {
1183             LOG.warn("Failed to get VPN interface for prefix {}", ipPrefix);
1184             return;
1185         }
1186
1187         String macAddress = FibUtil.getMacAddressFromPrefix(dataBroker, ifName, ipPrefix);
1188         if (macAddress == null) {
1189             LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix);
1190             return;
1191         }
1192
1193         actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(), new MacAddress(macAddress)));
1194     }
1195
1196     private void addTunnelInterfaceActions(String tunnelInterface, long vpnId, VrfEntry vrfEntry,
1197                                            List<ActionInfo> actionInfos) {
1198         Class<? extends TunnelTypeBase> tunnelType = getTunnelType(tunnelInterface);
1199         if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
1200             LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1201             actionInfos.add(new ActionPushMpls());
1202             actionInfos.add(new ActionSetFieldMplsLabel(vrfEntry.getLabel()));
1203             actionInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
1204         } else {
1205             int label = vrfEntry.getLabel().intValue();
1206             BigInteger tunnelId;
1207             // FIXME vxlan vni bit set is not working properly with OVS.need to
1208             // revisit
1209             if (tunnelType.equals(TunnelTypeVxlan.class)) {
1210                 tunnelId = BigInteger.valueOf(label);
1211             } else {
1212                 tunnelId = BigInteger.valueOf(label);
1213             }
1214
1215             LOG.debug("adding set tunnel id action for label {}", label);
1216             actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
1217             addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1218         }
1219     }
1220
1221     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1222         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1223         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1224         if (dpnInVpn.isPresent()) {
1225             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1226             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1227
1228             if (vpnInterfaces.remove(currVpnInterface)) {
1229                 if (vpnInterfaces.isEmpty()) {
1230                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1231                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1232                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
1233                 } else {
1234                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1235                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1236                         VpnInterfaces.class,
1237                         new VpnInterfacesKey(intfName)));
1238                 }
1239             }
1240         }
1241     }
1242
1243     private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
1244     /* Get interface info from prefix to interface mapping;
1245         Use the interface info to get the corresponding vpn interface op DS entry,
1246         remove the adjacency corresponding to this fib entry.
1247         If adjacency removed is the last adjacency, clean up the following:
1248          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1249          - prefix to interface entry
1250          - vpn interface op DS
1251      */
1252         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1253         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1254         Extraroute extraRoute = null;
1255         if (prefixInfo == null) {
1256             extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1257             if (extraRoute != null) {
1258                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1259                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1260
1261                     if (nextHopIp != null) {
1262                         prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32");
1263                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1264                     }
1265                 }
1266             }
1267             if (prefixInfo == null) {
1268                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1269                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1270                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1271                     PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1272                     prefixBuilder.setDpnId(lri.getDpnId());
1273                     prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1274                     prefixBuilder.setIpAddress(lri.getPrefix());
1275                     prefixInfo = prefixBuilder.build();
1276                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1277                         vrfEntry.getLabel(), prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1278                     checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1279                 }
1280             }
1281         } else {
1282             checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1283         }
1284     }
1285
1286     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1287                                           final VrfEntry vrfEntry, final Extraroute extraRoute) {
1288
1289         if (prefixInfo == null) {
1290             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1291             return; //Don't have any info for this prefix (shouldn't happen); need to return
1292         }
1293
1294         String ifName = prefixInfo.getVpnInterfaceName();
1295         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1296         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1297             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1298     }
1299
1300     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1301         Prefixes prefixInfo;
1302         Long vpnId;
1303         String rd;
1304         VrfEntry vrfEntry;
1305         Extraroute extraRoute;
1306
1307         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1308                                          final VrfEntry vrfEntry, final Extraroute extraRoute) {
1309             this.prefixInfo = prefixInfo;
1310             this.vpnId = vpnId;
1311             this.rd = rd;
1312             this.vrfEntry = vrfEntry;
1313             this.extraRoute = extraRoute;
1314         }
1315
1316         @Override
1317         public List<ListenableFuture<Void>> call() throws Exception {
1318             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1319             // to call the respective helpers.
1320
1321             //First Cleanup LabelRouteInfo
1322             synchronized (vrfEntry.getLabel().toString().intern()) {
1323                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1324                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1325                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1326                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1327                         FibUtil.getVpnInstanceOpData(dataBroker, rd);
1328                     String vpnInstanceName = "";
1329                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1330                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1331                     }
1332                     boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName);
1333                     if (lriRemoved) {
1334                         String parentRd = lri.getParentVpnRd();
1335                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1336                             FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1337                     }
1338                 } else {
1339                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1340                         FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1341                 }
1342             }
1343             String ifName = prefixInfo.getVpnInterfaceName();
1344             Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1345                 FibUtil.getVpnInterfaceIdentifier(ifName));
1346             if (optvpnInterface.isPresent()) {
1347                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1348                 if (vpnId != associatedVpnId) {
1349                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1350                         vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1351                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1352                     return null;
1353                 } else {
1354                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1355                         vrfEntry.getDestPrefix(), associatedVpnId);
1356                 }
1357             }
1358             if (extraRoute != null) {
1359                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1360                     FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1361             }
1362             Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1363                 FibUtil.getAdjListPath(ifName));
1364             int numAdj = 0;
1365             if (optAdjacencies.isPresent()) {
1366                 numAdj = optAdjacencies.get().getAdjacency().size();
1367             }
1368             //remove adjacency corr to prefix
1369             if (numAdj > 1) {
1370                 LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
1371                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1372                     FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1373             }
1374             if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
1375                 //clean up the vpn interface from DpnToVpn list
1376                 LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1377                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1378                     FibUtil.getVpnInterfaceIdentifier(ifName));
1379             }
1380             return null;
1381         }
1382     }
1383
1384     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1385         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1386         final String rd = vrfTableKey.getRouteDistinguisher();
1387         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1388         if (vpnInstance == null) {
1389             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1390             return;
1391         }
1392         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1393         long elanTag = 0L;
1394         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1395         if (subnetRoute != null) {
1396             elanTag = subnetRoute.getElantag();
1397             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1398                 rd, vrfEntry.getDestPrefix(), elanTag);
1399             if (vpnToDpnList != null) {
1400                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1401                 dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
1402                     () -> {
1403                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1404
1405                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1406
1407                             makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1408                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1409                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1410                                 makeLFibTableEntry(curDpn.getDpnId(), vrfEntry.getLabel(), null,
1411                                     DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1412                             }
1413                         }
1414                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1415                         futures.add(tx.submit());
1416                         return futures;
1417                     });
1418             }
1419             synchronized (vrfEntry.getLabel().toString().intern()) {
1420                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1421                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1422                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1423                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1424                         FibUtil.getVpnInstanceOpData(dataBroker, rd);
1425                     String vpnInstanceName = "";
1426                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1427                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1428                     }
1429                     boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName);
1430                     if (lriRemoved) {
1431                         String parentRd = lri.getParentVpnRd();
1432                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1433                             FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1434                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as "
1435                             + "labelRouteInfo cleared", vrfEntry.getLabel(), rd,
1436                             vrfEntry.getDestPrefix());
1437                     }
1438                 } else {
1439                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1440                         FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1441                     LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1442                         vrfEntry.getLabel(), rd, vrfEntry.getDestPrefix());
1443                 }
1444             }
1445             return;
1446         }
1447         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnInstance.getVpnId(), NwConstants.DEL_FLOW)) {
1448             return;
1449         }
1450
1451         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1452             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1453         if (vpnToDpnList != null) {
1454             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1455             dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
1456                 () -> {
1457                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1458
1459                     if (localDpnIdList.size() <= 0) {
1460                         for (VpnToDpnList curDpn : vpnToDpnList) {
1461                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1462                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1463                                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1464                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1465                                 }
1466                             } else {
1467                                 deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1468                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1469                             }
1470                         }
1471                     } else {
1472                         for (BigInteger localDpnId : localDpnIdList) {
1473                             for (VpnToDpnList curDpn : vpnToDpnList) {
1474                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1475                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1476                                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1477                                             deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1478                                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1479                                         }
1480                                     } else {
1481                                         deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1482                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1483                                     }
1484                                 }
1485                             }
1486                         }
1487                     }
1488                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1489                     futures.add(tx.submit());
1490                     return futures;
1491                 });
1492         }
1493
1494         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1495         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1496         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1497
1498         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1499         // of the interVpnLink.
1500         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(this.dataBroker, rd);
1501         if (optVpnUuid.isPresent()) {
1502             String vpnUuid = optVpnUuid.get();
1503             List<String> routeNexthoplist = vrfEntry.getNextHopAddressList();
1504             if (routeNexthoplist.isEmpty()) {
1505                 LOG.trace("NextHopList is empty for VrfEntry {}", vrfEntry);
1506                 return;
1507             }
1508             String routeNexthop = routeNexthoplist.get(0);
1509             Optional<InterVpnLinkDataComposite> optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1510             if (optInterVpnLink.isPresent()) {
1511                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1512                 if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1513                     // This is route that points to the other endpoint of an InterVpnLink
1514                     // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
1515                     removeInterVPNLinkRouteFlows(interVpnLink.getInterVpnLinkName(),
1516                         interVpnLink.isFirstEndpointVpnName(rd),
1517                         vrfEntry);
1518                 }
1519             }
1520         }
1521
1522     }
1523
1524     /*
1525       Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1526       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1527       provided by ResourceBatchingManager
1528      */
1529     private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
1530                                   final VrfEntry vrfEntry) {
1531         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1532
1533         final String rd = vrfTableKey.getRouteDistinguisher();
1534         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1535         if (vpnInstance == null) {
1536             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1537             return;
1538         }
1539         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1540         if (vpnToDpnList != null) {
1541             for (VpnToDpnList curDpn : vpnToDpnList) {
1542                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1543                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(),
1544                         vrfTableKey, vrfEntry, writeTx);
1545                 }
1546             }
1547         }
1548     }
1549
1550     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1551                                   final long vpnId, final VrfTablesKey vrfTableKey,
1552                                   final VrfEntry vrfEntry, WriteTransaction tx) {
1553
1554         Boolean wrTxPresent = true;
1555         if (tx == null) {
1556             wrTxPresent = false;
1557             tx = dataBroker.newWriteOnlyTransaction();
1558         }
1559
1560         LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
1561             vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
1562         String rd = vrfTableKey.getRouteDistinguisher();
1563
1564         if (localDpnId != null) {
1565             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1566             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1567             return;
1568         }
1569
1570         // below two reads are kept as is, until best way is found to identify dpnID
1571         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1572         Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1573
1574         if (localNextHopInfo == null && extraRoute != null) {
1575             // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1576             for (String nextHopIp : extraRoute.getNexthopIpList()) {
1577                 localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
1578                 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1579             }
1580         } else {
1581             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1582         }
1583         if (!wrTxPresent) {
1584             tx.submit();
1585         }
1586     }
1587
1588     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1589                                            VrfEntry vrfEntry, String rd, WriteTransaction tx) {
1590         boolean isRemoteRoute = true;
1591         if (localNextHopInfo != null) {
1592             isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId());
1593         }
1594         if (isRemoteRoute) {
1595             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1596             return true;
1597         } else {
1598             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}",
1599                 rd, vrfEntry.getDestPrefix(), remoteDpnId);
1600             return false;
1601         }
1602     }
1603
1604     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry, String rd, WriteTransaction tx) {
1605         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1606         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1607     }
1608
1609     private long get(byte[] rawIpAddress) {
1610         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1611             + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1612     }
1613
1614     private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1615                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1616         Boolean wrTxPresent = true;
1617         if (tx == null) {
1618             wrTxPresent = false;
1619             tx = dataBroker.newWriteOnlyTransaction();
1620         }
1621
1622         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1623         String[] values = vrfEntry.getDestPrefix().split("/");
1624         String ipAddress = values[0];
1625         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1626         if (addOrRemove == NwConstants.ADD_FLOW) {
1627             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1628         } else {
1629             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1630         }
1631         InetAddress destPrefix;
1632         try {
1633             destPrefix = InetAddress.getByName(ipAddress);
1634         } catch (UnknownHostException e) {
1635             LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1636             return;
1637         }
1638
1639         List<MatchInfo> matches = new ArrayList<>();
1640
1641         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
1642
1643         if (destPrefix instanceof Inet4Address) {
1644             matches.add(MatchEthernetType.IPV4);
1645             if (prefixLength != 0) {
1646                 matches.add(new MatchIpv4Destination(destPrefix.getHostAddress(), Integer.toString(prefixLength)));
1647             }
1648         } else {
1649             matches.add(MatchEthernetType.IPV6);
1650             if (prefixLength != 0) {
1651                 matches.add(new MatchIpv6Destination(destPrefix.getHostAddress() + "/" + prefixLength));
1652             }
1653         }
1654
1655         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1656         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1657         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority,
1658             flowRef, 0, 0,
1659             COOKIE_VM_FIB_TABLE, matches, instructions);
1660
1661         Flow flow = flowEntity.getFlowBuilder().build();
1662         String flowId = flowEntity.getFlowId();
1663         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1664         Node nodeDpn = buildDpnNode(dpId);
1665
1666         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1667             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1668             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1669
1670         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1671             SubTransaction subTransaction = new SubTransactionImpl();
1672             if (addOrRemove == NwConstants.ADD_FLOW) {
1673                 subTransaction.setInstanceIdentifier(flowInstanceId);
1674                 subTransaction.setInstance(flow);
1675                 subTransaction.setAction(SubTransaction.CREATE);
1676             } else {
1677                 subTransaction.setInstanceIdentifier(flowInstanceId);
1678                 subTransaction.setAction(SubTransaction.DELETE);
1679             }
1680             transactionObjects.add(subTransaction);
1681         }
1682
1683         if (addOrRemove == NwConstants.ADD_FLOW) {
1684             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
1685         } else {
1686             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1687         }
1688
1689         if (!wrTxPresent) {
1690             tx.submit();
1691         }
1692     }
1693
1694     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1695     private Node buildDpnNode(BigInteger dpnId) {
1696         NodeId nodeId = new NodeId("openflow:" + dpnId);
1697         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1698
1699         return nodeDpn;
1700     }
1701
1702     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1703                                     int addOrRemove, WriteTransaction tx) {
1704         Boolean wrTxPresent = true;
1705         if (tx == null) {
1706             wrTxPresent = false;
1707             tx = dataBroker.newWriteOnlyTransaction();
1708         }
1709
1710         List<MatchInfo> matches = new ArrayList<>();
1711         matches.add(MatchEthernetType.MPLS_UNICAST);
1712         matches.add(new MatchMplsLabel(label));
1713
1714         // Install the flow entry in L3_LFIB_TABLE
1715         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1716
1717         FlowEntity flowEntity;
1718         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1719             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1720         Flow flow = flowEntity.getFlowBuilder().build();
1721         String flowId = flowEntity.getFlowId();
1722         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1723         Node nodeDpn = buildDpnNode(dpId);
1724         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1725             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1726             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1727
1728         if (addOrRemove == NwConstants.ADD_FLOW) {
1729             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
1730         } else {
1731             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1732         }
1733         if (!wrTxPresent) {
1734             tx.submit();
1735         }
1736
1737         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1738             dpId, label, instructions, flowKey, (NwConstants.ADD_FLOW == addOrRemove) ? "ADDED" : "REMOVED");
1739     }
1740
1741     private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress,
1742                                       final String ipPrefixAddress) {
1743         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}", dpId, vpnId, ipAddress);
1744         try {
1745             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress);
1746         } catch (NullPointerException e) {
1747             LOG.trace("", e);
1748         }
1749     }
1750
1751     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1752                                     final FutureCallback<List<Void>> callback) {
1753         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1754         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1755         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1756         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1757         if (!vrfTable.isPresent()) {
1758             LOG.warn("VRF Table not yet available for RD {}", rd);
1759             if (callback != null) {
1760                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1761                 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1762                 Futures.addCallback(listenableFuture, callback);
1763             }
1764             return;
1765         }
1766         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1767         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1768             () -> {
1769                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1770                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1771                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1772                     for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1773                         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1774                         if (subnetRoute != null) {
1775                             long elanTag = subnetRoute.getElantag();
1776                             installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1777                             continue;
1778                         }
1779                         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1780                         if (routerInt != null) {
1781                             LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1782                                 rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1783                             installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
1784                                 routerInt.getIpAddress(),
1785                                 new MacAddress(routerInt.getMacAddress()), NwConstants.ADD_FLOW);
1786                             continue;
1787                         }
1788                         //Handle local flow creation for imports
1789                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1790                             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1791                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1792                                 && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1793                                 if (lri.getDpnId().equals(dpnId)) {
1794                                     createLocalFibEntry(vpnId, rd, vrfEntry);
1795                                     continue;
1796                                 }
1797                             }
1798                         }
1799                         // Passing null as we don't know the dpn
1800                         // to which prefix is attached at this point
1801                         createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1802                     }
1803                     //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1804                     futures.add(tx.submit());
1805                 }
1806                 if (callback != null) {
1807                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1808                     Futures.addCallback(listenableFuture, callback);
1809                 }
1810                 return futures;
1811             });
1812     }
1813
1814
1815     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1816                                             final String localNextHopIp, final String remoteNextHopIp) {
1817         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1818             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1819         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1820         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1821         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1822         if (vrfTable.isPresent()) {
1823             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1824             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1825                 () -> {
1826                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1827                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1828                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1829                         for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1830                             if (!vrfEntry.getNextHopAddressList().isEmpty()) {
1831                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1832                                     if (remoteNextHopIp.trim()
1833                                         .equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1834                                         LOG.trace(" creating remote FIB entry for prefix {} rd {}",
1835                                             vrfEntry.getDestPrefix(), rd);
1836                                         createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(),
1837                                             vrfEntry, writeCfgTxn);
1838                                     }
1839                                 }
1840                             }
1841                         }
1842                         futures.add(writeCfgTxn.submit());
1843                     }
1844                     return futures;
1845                 });
1846         }
1847     }
1848
1849     public void populateInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1850                                             final String localNextHopIp, final String remoteNextHopIp) {
1851         LOG.trace("populateInternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1852             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1853         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1854         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1855         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1856         if (vrfTable.isPresent()) {
1857             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1858             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1859                 () -> {
1860                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1861                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1862                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1863                         for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1864                             // Handle Internal Routes only (i.e., STATIC for now)
1865                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC) {
1866                                 SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1867                                     /* Ignore SubnetRoute entry */
1868                                 if (subnetRoute == null) {
1869                                     if (!vrfEntry.getNextHopAddressList().isEmpty()) {
1870                                         if (remoteNextHopIp.trim()
1871                                             .equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1872                                             LOG.trace(" creating remote FIB entry for prefix {} rd {} on Dpn {}",
1873                                                 vrfEntry.getDestPrefix(), rd, dpnId);
1874                                             createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry,
1875                                                 writeCfgTxn);
1876                                         }
1877                                     }
1878                                 }
1879                             }
1880                         }
1881                         futures.add(writeCfgTxn.submit());
1882                     }
1883                     return futures;
1884                 });
1885         }
1886     }
1887
1888     public void manageRemoteRouteOnDPN(final boolean action,
1889                                        final BigInteger localDpnId,
1890                                        final long vpnId,
1891                                        final String rd,
1892                                        final String destPrefix,
1893                                        final String destTepIp) {
1894         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1895
1896         if (vpnInstance == null) {
1897             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1898             return;
1899         }
1900         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1901         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + localDpnId.toString(),
1902             () -> {
1903                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1904                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1905                     WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1906                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1907                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1908                     if (vrfEntry == null) {
1909                         return futures;
1910                     }
1911                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1912                         action, localDpnId, vpnId, rd, destPrefix);
1913                     List<String> nextHopAddressList = vrfEntry.getNextHopAddressList();
1914                     VrfEntry modVrfEntry;
1915                     if (nextHopAddressList == null || (nextHopAddressList.isEmpty())) {
1916                         List<String> nhList = Arrays.asList(destTepIp);
1917                         modVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(nhList).build();
1918                     } else {
1919                         modVrfEntry = vrfEntry;
1920                     }
1921
1922                     if (action == true) {
1923                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1924                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
1925                     } else {
1926                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1927                         deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
1928                     }
1929                     futures.add(writeTransaction.submit());
1930                 }
1931                 return futures;
1932             });
1933     }
1934
1935     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1936                                  final FutureCallback<List<Void>> callback) {
1937         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1938         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1939         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1940         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1941         if (vrfTable.isPresent()) {
1942             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1943             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1944                 () -> {
1945                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1946                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1947                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1948                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1949                                 /* Handle subnet routes here */
1950                             SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1951                             if (subnetRoute != null) {
1952                                 LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn",
1953                                     vrfEntry.getDestPrefix(),
1954                                     dpnId, rd);
1955                                 makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1956                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY,
1957                                     NwConstants.DEL_FLOW, tx);
1958                                 LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}",
1959                                     vrfEntry.getLabel(), rd,
1960                                     vrfEntry.getDestPrefix());
1961                                 continue;
1962                             }
1963                             // ping responder for router interfaces
1964                             RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1965                             if (routerInt != null) {
1966                                 LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1967                                     rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1968                                 installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
1969                                     routerInt.getIpAddress(),
1970                                     new MacAddress(routerInt.getMacAddress()), NwConstants.DEL_FLOW);
1971                                 continue;
1972                             }
1973                             // Passing null as we don't know the dpn
1974                             // to which prefix is attached at this point
1975                             deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1976                         }
1977                         futures.add(tx.submit());
1978                         if (callback != null) {
1979                             ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1980                             Futures.addCallback(listenableFuture, callback);
1981                         }
1982                     }
1983                     return futures;
1984                 });
1985         }
1986     }
1987
1988     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1989                                            final String localNextHopIp, final String remoteNextHopIp) {
1990         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1991                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1992             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1993         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1994         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1995         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1996         if (vrfTable.isPresent()) {
1997             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1998             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1999                 () -> {
2000                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2001                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2002                         WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
2003                         for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
2004                             if (!vrfEntry.getNextHopAddressList().isEmpty()) {
2005                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2006                                     if (remoteNextHopIp.trim()
2007                                         .equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
2008                                         LOG.trace(" deleting remote FIB entry {}", vrfEntry);
2009                                         deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(),
2010                                             vrfEntry, writeTransaction);
2011                                     }
2012                                 }
2013                             }
2014                         }
2015                         futures.add(writeTransaction.submit());
2016                     }
2017                     return futures;
2018                 });
2019         }
2020     }
2021
2022     public void cleanUpInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
2023                                            final String localNextHopIp, final String remoteNextHopIp) {
2024         LOG.trace("cleanUpInternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
2025                 + " localNexthopIp {} , remoteNexhtHopIp {}",
2026             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2027         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2028         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2029         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2030         if (vrfTable.isPresent()) {
2031             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2032             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2033                 () -> {
2034                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2035                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2036                         WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
2037                         for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
2038                             // Handle Internal Routes only (i.e, STATIC for now)
2039                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC) {
2040                                 SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
2041                                     /* Ignore SubnetRoute entry */
2042                                 if (subnetRoute == null) {
2043                                     if (!vrfEntry.getNextHopAddressList().isEmpty()) {
2044                                         if (remoteNextHopIp.trim()
2045                                             .equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
2046                                             LOG.trace(" deleting remote FIB entry {}", vrfEntry);
2047                                             deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(),
2048                                                 vrfEntry, writeTransaction);
2049                                         }
2050                                     }
2051                                 }
2052                             }
2053                         }
2054                         futures.add(writeTransaction.submit());
2055                     }
2056                     return futures;
2057                 });
2058         }
2059     }
2060
2061     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
2062         InstanceIdentifierBuilder<VrfTables> idBuilder =
2063             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
2064         InstanceIdentifier<VrfTables> id = idBuilder.build();
2065         return id;
2066     }
2067
2068     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
2069         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2070             .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(label).append(NwConstants.FLOWID_SEPARATOR)
2071             .append(priority).toString();
2072     }
2073
2074     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
2075         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2076             .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
2077             .append(rd).append(NwConstants.FLOWID_SEPARATOR)
2078             .append(priority).append(NwConstants.FLOWID_SEPARATOR)
2079             .append(destPrefix.getHostAddress()).toString();
2080     }
2081
2082     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
2083         return new StringBuilder(64).append(FLOWID_PREFIX)
2084             .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
2085             .append(prefix).append(NwConstants.FLOWID_SEPARATOR)
2086             .append(nextHop).toString();
2087     }
2088
2089     protected List<AdjacencyResult> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
2090                                                      final VrfEntry vrfEntry, String rd) {
2091         List<AdjacencyResult> adjacencyList = new ArrayList<>();
2092         List<String> prefixIpList = new ArrayList<>();
2093         LOG.trace("resolveAdjacency called with remotedDpnId {}, vpnId{}, VrfEntry {}",
2094             remoteDpnId, vpnId, vrfEntry);
2095         try {
2096             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2097                 Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
2098                 if (extraRoute == null) {
2099                     prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
2100                 } else {
2101                     prefixIpList = new ArrayList<>();
2102                     for (String extraRouteIp : extraRoute.getNexthopIpList()) {
2103                         prefixIpList.add(extraRouteIp + "/32");
2104                     }
2105                 }
2106             } else {
2107                 prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
2108             }
2109
2110             for (String prefixIp : prefixIpList) {
2111                 for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
2112                     LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
2113                     AdjacencyResult adjacencyResult = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
2114                         prefixIp, nextHopIp);
2115                     if (adjacencyResult != null && !adjacencyList.contains(adjacencyResult)) {
2116                         adjacencyList.add(adjacencyResult);
2117                     }
2118                 }
2119             }
2120         } catch (NullPointerException e) {
2121             LOG.trace("", e);
2122         }
2123         return adjacencyList;
2124     }
2125
2126     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
2127         InstanceIdentifier<VpnInstanceOpDataEntry> id =
2128             InstanceIdentifier.create(VpnInstanceOpData.class)
2129                 .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
2130         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
2131             FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
2132         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
2133     }
2134
2135     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2136         return new StringBuffer().append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2137             .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(tableMiss)
2138             .append(FLOWID_PREFIX).toString();
2139     }
2140
2141     /*
2142      * Install flow entry in protocol table to forward mpls
2143      * coming through gre tunnel to LFIB table.
2144      */
2145     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
2146         final BigInteger cookieProtocolTable = new BigInteger("1070000", 16);
2147         // Instruction to goto L3 InterfaceTable
2148         List<InstructionInfo> instructions = new ArrayList<>();
2149         instructions.add(new InstructionGotoTable(NwConstants.L3_LFIB_TABLE));
2150         List<MatchInfo> matches = new ArrayList<>();
2151         matches.add(MatchEthernetType.MPLS_UNICAST);
2152         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2153             getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2154                 NwConstants.L3_LFIB_TABLE),
2155             DEFAULT_FIB_FLOW_PRIORITY,
2156             "Protocol Table For LFIB",
2157             0, 0,
2158             cookieProtocolTable,
2159             matches, instructions);
2160
2161         if (addOrRemove == NwConstants.ADD_FLOW) {
2162             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2163             mdsalManager.installFlow(flowEntityToLfib);
2164         } else {
2165             mdsalManager.removeFlow(flowEntityToLfib);
2166         }
2167     }
2168
2169     public List<String> printFibEntries() {
2170         List<String> result = new ArrayList<>();
2171         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2172         result.add("-------------------------------------------------------------------");
2173         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2174         Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2175         if (fibEntries.isPresent()) {
2176             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2177             for (VrfTables vrfTable : vrfTables) {
2178                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2179                     for (String nextHop : vrfEntry.getNextHopAddressList()) {
2180                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2181                             vrfTable.getRouteDistinguisher(),
2182                             vrfEntry.getDestPrefix(), nextHop, vrfEntry.getLabel(), vrfEntry.getOrigin()));
2183                     }
2184                     if (vrfEntry.getNextHopAddressList().isEmpty()) {
2185                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2186                             vrfTable.getRouteDistinguisher(),
2187                             vrfEntry.getDestPrefix(), "local", vrfEntry.getLabel(), vrfEntry.getOrigin()));
2188                     }
2189                 }
2190             }
2191         }
2192         return result;
2193     }
2194
2195
2196     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2197         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2198             .child(VrfTables.class, new VrfTablesKey(rd))
2199             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2200         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2201         if (vrfEntry.isPresent()) {
2202             return vrfEntry.get();
2203         }
2204         return null;
2205     }
2206
2207     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2208         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2209             .child(VrfTables.class, new VrfTablesKey(rd))
2210             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2211         return vrfEntryId;
2212     }
2213
2214     protected Boolean isIpv4Address(String ipAddress) {
2215         try {
2216             InetAddress address = InetAddress.getByName(ipAddress);
2217             if (address instanceof Inet4Address) {
2218                 return true;
2219             }
2220         } catch (UnknownHostException e) {
2221             LOG.warn("Invalid ip address {}", ipAddress, e);
2222             return false;
2223         }
2224         return false;
2225     }
2226
2227     protected Boolean installRouterFibEntries(final VrfEntry vrfEntry, final Collection<VpnToDpnList> vpnToDpnList,
2228                                               long vpnId, int addOrRemove) {
2229         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
2230         if (routerInt != null && vpnToDpnList != null) {
2231             String routerId = routerInt.getUuid();
2232             String macAddress = routerInt.getMacAddress();
2233             String ipValue = routerInt.getIpAddress();
2234             LOG.trace("createFibEntries - Router augmented vrfentry found for for router uuid:{}, ip:{}, mac:{}",
2235                 routerId, ipValue, macAddress);
2236             for (VpnToDpnList vpnDpn : vpnToDpnList) {
2237                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
2238                     installRouterFibEntry(vrfEntry, vpnDpn.getDpnId(), vpnId, routerId, ipValue,
2239                         new MacAddress(macAddress), addOrRemove);
2240                 }
2241             }
2242             return true;
2243         }
2244         return false;
2245     }
2246
2247     public void installRouterFibEntry(final VrfEntry vrfEntry, BigInteger dpnId, long vpnId, String routerUuid,
2248                                       String routerInternalIp, MacAddress routerMac, int addOrRemove) {
2249         String[] subSplit = routerInternalIp.split("/");
2250         if (!isIpv4Address(subSplit[0])) {
2251             // Ping responder using OpenFlow rules is only supported for IPv4, hence skipping.
2252             return;
2253         }
2254
2255         String addRemoveStr = (addOrRemove == NwConstants.ADD_FLOW) ? "ADD_FLOW" : "DELETE_FLOW";
2256         LOG.trace("{}: bulding Echo Flow entity for dpid:{}, router_ip:{}, vpnId:{}, subSplit:{} ", addRemoveStr,
2257             dpnId, routerInternalIp, vpnId, subSplit[0]);
2258
2259         List<MatchInfo> matches = new ArrayList<>();
2260
2261         matches.add(MatchIpProtocol.ICMP);
2262         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
2263         matches.add(new MatchIcmpv4((short) 8, (short) 0));
2264         matches.add(MatchEthernetType.IPV4);
2265         matches.add(new MatchIpv4Destination(subSplit[0], "32"));
2266
2267         List<ActionInfo> actionsInfos = new ArrayList<>();
2268
2269         // Set Eth Src and Eth Dst
2270         actionsInfos.add(new ActionMoveSourceDestinationEth());
2271         actionsInfos.add(new ActionSetFieldEthernetSource(routerMac));
2272
2273         // Move Ip Src to Ip Dst
2274         actionsInfos.add(new ActionMoveSourceDestinationIp());
2275         actionsInfos.add(new ActionSetSourceIp(subSplit[0], "32"));
2276
2277         // Set the ICMP type to 0 (echo reply)
2278         actionsInfos.add(new ActionSetIcmpType((short) 0));
2279
2280         actionsInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
2281
2282         actionsInfos.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2283
2284         List<InstructionInfo> instructions = new ArrayList<>();
2285
2286         instructions.add(new InstructionApplyActions(actionsInfos));
2287
2288         int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH;
2289         String flowRef = getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, vrfEntry.getLabel(), priority);
2290
2291         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef,
2292             0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions);
2293
2294         if (addOrRemove == NwConstants.ADD_FLOW) {
2295             mdsalManager.installFlow(flowEntity);
2296         } else {
2297             mdsalManager.removeFlow(flowEntity);
2298         }
2299     }
2300
2301     public void removeInterVPNLinkRouteFlows(final String interVpnLinkName,
2302                                              final boolean isVpnFirstEndPoint,
2303                                              final VrfEntry vrfEntry) {
2304         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
2305             && vrfEntry.getNextHopAddressList().size() == 1);
2306         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLinkName);
2307
2308         if (!interVpnLinkState.isPresent()) {
2309             LOG.warn("Could not find State for InterVpnLink {}", interVpnLinkName);
2310             return;
2311         }
2312
2313         List<BigInteger> targetDpns =
2314             isVpnFirstEndPoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
2315                 : interVpnLinkState.get().getSecondEndpointState().getDpId();
2316
2317         String nextHop = vrfEntry.getNextHopAddressList().get(0);
2318
2319         // delete from FIB
2320         //
2321         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2322         FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2323         Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef)).setTableId(NwConstants.L3_FIB_TABLE)
2324             .setFlowName(flowRef).build();
2325
2326         LOG.trace("Removing flow in FIB table for interVpnLink {} key {}",
2327             interVpnLinkName, flowRef);
2328
2329         for (BigInteger dpId : targetDpns) {
2330             LOG.debug("Removing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2331                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), nextHop,
2332                 dpId, interVpnLinkName);
2333
2334             mdsalManager.removeFlow(dpId, flow);
2335         }
2336
2337         // delete from LFIB
2338         //
2339         LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2340
2341         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
2342         for (BigInteger dpId : targetDpns) {
2343             LOG.debug("Removing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
2344                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), nextHop,
2345                 dpId, interVpnLinkName);
2346             makeLFibTableEntry(dpId, vrfEntry.getLabel(), null /* no instructions */,
2347                 LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, tx);
2348         }
2349         tx.submit();
2350     }
2351 }