vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.util.concurrent.FutureCallback;
  13 import com.google.common.util.concurrent.Futures;
  14 import com.google.common.util.concurrent.ListenableFuture;
  15 import java.math.BigInteger;
  16 import java.net.Inet4Address;
  17 import java.net.InetAddress;
  18 import java.net.UnknownHostException;
  19 import java.util.ArrayList;
  20 import java.util.Arrays;
  21 import java.util.Collection;
  22 import java.util.Collections;
  23 import java.util.List;
  24 import java.util.concurrent.BlockingQueue;
  25 import java.util.concurrent.Callable;
  26 import java.util.concurrent.ExecutionException;
  27 import java.util.concurrent.Future;
  28 import java.util.concurrent.LinkedBlockingQueue;
  29 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  30 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  31 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  32 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  33 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  34 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  35 import org.opendaylight.genius.mdsalutil.ActionInfo;
  36 import org.opendaylight.genius.mdsalutil.FlowEntity;
  37 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  38 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  39 import org.opendaylight.genius.mdsalutil.MatchInfo;
  40 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  41 import org.opendaylight.genius.mdsalutil.NwConstants;
  42 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  43 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationEth;
  44 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationIp;
  45 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionSetIcmpType;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionSetSourceIp;
  55 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  56 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  57 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  58 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  59 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  60 import org.opendaylight.genius.mdsalutil.matches.MatchIcmpv4;
  61 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  62 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  63 import org.opendaylight.genius.mdsalutil.matches.MatchIpv6Destination;
  64 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  65 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  67 import org.opendaylight.genius.utils.ServiceIndex;
  68 import org.opendaylight.genius.utils.batching.ActionableResource;
  69 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  70 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  71 import org.opendaylight.genius.utils.batching.ResourceHandler;
  72 import org.opendaylight.genius.utils.batching.SubTransaction;
  73 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
  74 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  75 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  76 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  77 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  78 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  79 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  80 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.Tunnel;
  81 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeInputBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeOutput;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroute;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.Vpn;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.VpnKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 135 import org.opendaylight.yangtools.yang.binding.DataObject;
 136 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 137 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 138 import org.opendaylight.yangtools.yang.common.RpcResult;
 139 import org.slf4j.Logger;
 140 import org.slf4j.LoggerFactory;
 141
 142 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener>
 143     implements AutoCloseable, ResourceHandler {
 144
 145     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 146     private static final String FLOWID_PREFIX = "L3.";
 147     private final DataBroker dataBroker;
 148     private final IMdsalApiManager mdsalManager;
 149     private IVpnManager vpnmanager;
 150     private final NexthopManager nextHopManager;
 151     private ItmRpcService itmManager;
 152     private final OdlInterfaceRpcService interfaceManager;
 153     private final IdManagerService idManager;
 154     private static final BigInteger COOKIE_VM_FIB_TABLE = new BigInteger("8000003", 16);
 155     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 156     private static final int LFIB_INTERVPN_PRIORITY = 1;
 157     private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
 158     private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
 159     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 160     List<SubTransaction> transactionObjects;
 161     private static final int PERIODICITY = 500;
 162     private static Integer batchSize;
 163     private static Integer batchInterval;
 164     private static final int BATCH_SIZE = 1000;
 165     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 166     private final ResourceBatchingManager resourceBatchingManager;
 167
 168     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 169                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
 170                             final IdManagerService idManager) {
 171         super(VrfEntry.class, VrfEntryListener.class);
 172         this.dataBroker = dataBroker;
 173         this.mdsalManager = mdsalApiManager;
 174         this.nextHopManager = nexthopManager;
 175         this.interfaceManager = interfaceManager;
 176         this.idManager = idManager;
 177
 178         batchSize = Integer.getInteger("batch.size");
 179         if (batchSize == null) {
 180             batchSize = BATCH_SIZE;
 181         }
 182         batchInterval = Integer.getInteger("batch.wait.time");
 183         if (batchInterval == null) {
 184             batchInterval = PERIODICITY;
 185         }
 186         resourceBatchingManager = ResourceBatchingManager.getInstance();
 187         resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
 188         transactionObjects = new ArrayList<>();
 189     }
 190
 191     public void start() {
 192         LOG.info("{} start", getClass().getSimpleName());
 193         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 194     }
 195
 196     @Override
 197     protected VrfEntryListener getDataTreeChangeListener() {
 198         return VrfEntryListener.this;
 199     }
 200
 201     @Override
 202     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 203         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 204     }
 205
 206     @Override
 207     public DataBroker getResourceBroker() {
 208         return dataBroker;
 209     }
 210
 211     @Override
 212     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 213         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 214         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 215         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} nexthop {} label {}",
 216             rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 217         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 218             createFibEntries(identifier, vrfEntry);
 219         } else {
 220             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 221             actResource.setAction(ActionableResource.CREATE);
 222             actResource.setInstanceIdentifier(identifier);
 223             actResource.setInstance(vrfEntry);
 224             vrfEntryBufferQ.add(actResource);
 225         }
 226         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 227         LOG.info("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
 228             rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 229     }
 230
 231     @Override
 232     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 233         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 234         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 235         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} nexthop {} label {}",
 236             rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 237         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 238             deleteFibEntries(identifier, vrfEntry);
 239         } else {
 240             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 241             actResource.setAction(ActionableResource.DELETE);
 242             actResource.setInstanceIdentifier(identifier);
 243             actResource.setInstance(vrfEntry);
 244             vrfEntryBufferQ.add(actResource);
 245         }
 246         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 247         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
 248             rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 249     }
 250
 251     @Override
 252     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 253         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 254
 255         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 256         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
 257         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
 258             rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 259         // Handle BGP Routes first
 260         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 261             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + update.getDestPrefix());
 262             actResource.setAction(ActionableResource.UPDATE);
 263             actResource.setInstanceIdentifier(identifier);
 264             actResource.setInstance(update);
 265             actResource.setOldInstance(original);
 266             vrfEntryBufferQ.add(actResource);
 267             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 268                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 269             return;
 270         }
 271
 272         // Handle Internal Routes next (ie., STATIC only)
 273         if (FibUtil.isControllerManagedNonInterVpnLinkRoute(RouteOrigin.value(update.getOrigin()))) {
 274             SubnetRoute subnetRoute = update.getAugmentation(SubnetRoute.class);
 275             /* Ignore SubnetRoute entry, as it will be driven by createFibEntries call down below */
 276             if (subnetRoute == null) {
 277                 List<String> origNhList = original.getNextHopAddressList();
 278                 List<String> updateNhList = update.getNextHopAddressList();
 279                 //final SubnetRoute subnetRoute = update.getAugmentation(SubnetRoute.class);
 280                 LOG.info("UPDATE: Original nexthop {} updateNextHop {} ", origNhList, updateNhList);
 281
 282                 // If original VRF Entry had nexthop null , but update VRF Entry
 283                 // has nexthop , route needs to be created on remote Dpns
 284                 if (((origNhList == null) || (origNhList.isEmpty())
 285                     && (updateNhList != null) && (!updateNhList.isEmpty()))) {
 286                     // TODO(vivek): Though ugly, Not handling this code now, as each
 287                     // tep add event will invoke flow addition
 288                     LOG.trace("Original VRF entry NH is null for destprefix {}. This event is IGNORED here.",
 289                         update.getDestPrefix());
 290                     return;
 291                 }
 292
 293                 // If original VRF Entry had valid nexthop , but update VRF Entry
 294                 // has nexthop empty'ed out, route needs to be removed from remote Dpns
 295                 if (((updateNhList == null) || (updateNhList.isEmpty())
 296                     && (origNhList != null) && (!origNhList.isEmpty()))) {
 297                     LOG.trace("Original VRF entry had valid NH for destprefix {}. This event is IGNORED here.",
 298                         update.getDestPrefix());
 299                     return;
 300                 }
 301             }
 302             createFibEntries(identifier, update);
 303             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 304                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 305             return;
 306         }
 307
 308         /* Handl all other route origins */
 309         createFibEntries(identifier, update);
 310
 311         LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 312             rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 313     }
 314
 315     @Override
 316     public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 317                        Object original, Object update, List<SubTransaction> transactionObjects) {
 318         this.transactionObjects = transactionObjects;
 319         if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 320             createFibEntries(tx, identifier, (VrfEntry) update);
 321         }
 322     }
 323
 324     @Override
 325     public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 326                        Object vrfEntry, List<SubTransaction> transactionObjects) {
 327         this.transactionObjects = transactionObjects;
 328         if (vrfEntry instanceof VrfEntry) {
 329             createFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 330         }
 331     }
 332
 333     @Override
 334     public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 335                        Object vrfEntry, List<SubTransaction> transactionObjects) {
 336         this.transactionObjects = transactionObjects;
 337         if (vrfEntry instanceof VrfEntry) {
 338             deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 339         }
 340     }
 341
 342     @Override
 343     public int getBatchSize() {
 344         return batchSize;
 345     }
 346
 347     @Override
 348     public int getBatchInterval() {
 349         return batchInterval;
 350     }
 351
 352     @Override
 353     public LogicalDatastoreType getDatastoreType() {
 354         return LogicalDatastoreType.CONFIGURATION;
 355     }
 356
 357     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 358         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 359
 360         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 361         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 362         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 363             + " has null vpnId!");
 364
 365         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 366         final Long vpnId = vpnInstance.getVpnId();
 367         final String rd = vrfTableKey.getRouteDistinguisher();
 368         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 369         if (subnetRoute != null) {
 370             final long elanTag = subnetRoute.getElantag();
 371             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 372                 rd, vrfEntry.getDestPrefix(), elanTag);
 373             if (vpnToDpnList != null) {
 374                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 375                 dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
 376                     new Callable<List<ListenableFuture<Void>>>() {
 377                         @Override
 378                         public List<ListenableFuture<Void>> call() throws Exception {
 379                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 380                             for (final VpnToDpnList curDpn : vpnToDpnList) {
 381                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 382                                     installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId.longValue(),
 383                                         vrfEntry, tx);
 384                                 }
 385                             }
 386                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 387                             futures.add(tx.submit());
 388                             return futures;
 389                         }
 390                     });
 391             }
 392             return;
 393         }
 394         // ping responder for router interfaces
 395         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnId, NwConstants.ADD_FLOW)) {
 396             return;
 397         }
 398
 399         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 400
 401         if (vpnToDpnList != null) {
 402             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 403             dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
 404                 new Callable<List<ListenableFuture<Void>>>() {
 405                     @Override
 406                     public List<ListenableFuture<Void>> call() throws Exception {
 407                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 408                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 409                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 410                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 411                                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 412                                         vrfTableKey, vrfEntry, tx);
 413                                 }
 414                             }
 415                         }
 416                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 417                         futures.add(tx.submit());
 418                         return futures;
 419                     }
 420                 });
 421         }
 422
 423         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 424         if (optVpnUuid.isPresent()) {
 425             Optional<InterVpnLinkDataComposite> optInterVpnLink =
 426                 InterVpnLinkCache.getInterVpnLinkByVpnId(optVpnUuid.get());
 427             LOG.debug("InterVpnLink {} found in Cache: {}", optVpnUuid.get(), optInterVpnLink.isPresent());
 428             if (optInterVpnLink.isPresent()) {
 429                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
 430                 String vpnUuid = optVpnUuid.get();
 431                 String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
 432                 if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 433                     // This is an static route that points to the other endpoint of an InterVpnLink
 434                     // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 435                     installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 436                     installInterVpnRouteInLFib(rd, vrfEntry);
 437                 }
 438             }
 439         }
 440     }
 441
 442
 443     /*
 444       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 445       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 446       provided by ResourceBatchingManager
 447      */
 448     private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
 449                                   final VrfEntry vrfEntry) {
 450         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 451
 452         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 453         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 454         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 455             + " has null vpnId!");
 456
 457         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 458         final String rd = vrfTableKey.getRouteDistinguisher();
 459         if (vpnToDpnList != null) {
 460             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 461                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 462                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 463                 }
 464             }
 465         }
 466     }
 467
 468     private boolean originMustBeLeaked(InterVpnLink ivpnLink, RouteOrigin routeOrigin) {
 469         return routeOrigin == RouteOrigin.BGP && ivpnLink.isBgpRoutesLeaking()
 470             || routeOrigin == RouteOrigin.STATIC && ivpnLink.isStaticRoutesLeaking()
 471             || routeOrigin == RouteOrigin.CONNECTED && ivpnLink.isConnectedRoutesLeaking();
 472     }
 473
 474     // FIXME: Refactoring needed here.
 475     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 476     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 477                                    int addOrRemove) {
 478         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 479         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 480
 481         String rd = vrfTableKey.getRouteDistinguisher();
 482         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 483         if (vpnInstance == null) {
 484             LOG.error("VPN Instance not available for route with prefix {} label {} nextHop {} RD {}. Returning...",
 485                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
 486             return;
 487         }
 488         String vpnUuid = vpnInstance.getVpnInstanceName();
 489         Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(),
 490             "Could not find suitable VPN UUID for Route-Distinguisher=" + rd);
 491
 492         // if the new vrfEntry has been learned by Quagga BGP, its necessary to check if it's
 493         // there an interVpnLink for the involved vpn in order to make learn the new route to
 494         // the other part of the inter-vpn-link.
 495
 496         // For leaking, we need the InterVpnLink to be active. For removal, we just need a InterVpnLink.
 497         Optional<InterVpnLink> interVpnLink =
 498             (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(dataBroker, rd)
 499                 : FibUtil.getInterVpnLinkByRd(dataBroker, rd);
 500         if (!interVpnLink.isPresent()) {
 501             LOG.debug("Could not find an InterVpnLink for Route-Distinguisher={}", rd);
 502             return;
 503         }
 504
 505         // Ok, at this point everything is ready for the leaking/removal... but should it be performed?
 506         // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled.
 507         boolean proceed =
 508             addOrRemove == NwConstants.DEL_FLOW || originMustBeLeaked(interVpnLink.get(),
 509                 RouteOrigin.value(vrfEntry.getOrigin()));
 510
 511         if (proceed) {
 512             boolean isVpnFirstEndpoint = interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid);
 513
 514             String theOtherVpnId = isVpnFirstEndpoint ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
 515                 : interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue();
 516             String dstVpnRd = FibUtil.getVpnRd(dataBroker, theOtherVpnId);
 517             String endpointIp = isVpnFirstEndpoint ? interVpnLink.get().getFirstEndpoint().getIpAddress().getValue()
 518                 : interVpnLink.get().getSecondEndpoint().getIpAddress().getValue();
 519
 520             InstanceIdentifier<VrfEntry> vrfEntryIidInOtherVpn =
 521                 InstanceIdentifier.builder(FibEntries.class)
 522                     .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
 523                     .child(VrfEntry.class, new VrfEntryKey(vrfEntry.getDestPrefix()))
 524                     .build();
 525             if (addOrRemove == NwConstants.ADD_FLOW) {
 526                 LOG.debug("Leaking route (destination={}, nexthop={}) from Vrf={} to Vrf={}",
 527                     vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), rd, dstVpnRd);
 528                 String key = rd + FibConstants.SEPARATOR + vrfEntry.getDestPrefix();
 529                 long label = FibUtil.getUniqueId(idManager, FibConstants.VPN_IDPOOL_NAME, key);
 530                 VrfEntry newVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(Arrays.asList(endpointIp))
 531                     .setLabel(label)
 532                     .setOrigin(RouteOrigin.INTERVPN.getValue())
 533                     .build();
 534                 MDSALUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
 535                     vrfEntryIidInOtherVpn, newVrfEntry);
 536             } else {
 537                 LOG.debug("Removing leaked vrfEntry={}", vrfEntryIidInOtherVpn.toString());
 538                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn);
 539             }
 540         }
 541     }
 542
 543     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 544         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 545         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 546         prefixBuilder.setDpnId(lri.getDpnId());
 547         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 548         prefixBuilder.setIpAddress(lri.getPrefix());
 549         // Increment the refCount here
 550         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 551             .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) lri.getLabel())).build();
 552         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 553         if (!isPresentInList) {
 554             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 555             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 556             vpnInstanceNames.add(vpnInstanceName);
 557             builder.setVpnInstanceList(vpnInstanceNames);
 558             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(),
 559                 FibUtil.DEFAULT_CALLBACK);
 560         } else {
 561             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 562         }
 563         return prefixBuilder.build();
 564     }
 565
 566     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 567                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 568         Boolean wrTxPresent = true;
 569         if (tx == null) {
 570             wrTxPresent = false;
 571             tx = dataBroker.newWriteOnlyTransaction();
 572         }
 573         synchronized (vrfEntry.getLabel().toString().intern()) {
 574             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 575             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
 576                 && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 577
 578                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 579                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 580                         FibUtil.getVpnInstanceOpData(dataBroker, rd);
 581                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
 582                         String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 583                         if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 584                             updateVpnReferencesInLri(lri, vpnInstanceName, false);
 585                         }
 586                     }
 587                 }
 588                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 589                     vrfEntry.getLabel(), lri.getVpnInterfaceName(), lri.getDpnId());
 590             }
 591         }
 592         final List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 593         BigInteger subnetRouteMeta = ((BigInteger.valueOf(elanTag)).shiftLeft(32))
 594             .or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 595         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 596         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 597         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 598
 599         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 600             List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 601             // reinitialize instructions list for LFIB Table
 602             final List<InstructionInfo> LFIBinstructions = new ArrayList<InstructionInfo>();
 603
 604             actionsInfos.add(new ActionPopMpls());
 605             LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 606             LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 607                 MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 608             LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 609
 610             makeLFibTableEntry(dpnId, vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 611                 NwConstants.ADD_FLOW, tx);
 612         }
 613         if (!wrTxPresent) {
 614             tx.submit();
 615         }
 616     }
 617
 618     /*
 619      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 620      * LportDispatcher table (via table 80)
 621      */
 622     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 623         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 624         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 625         // packet is commuted from Vpn2 to Vpn1.
 626         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
 627         if (!vpnNameOpc.isPresent()) {
 628             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 629             return;
 630         }
 631
 632         String vpnName = vpnNameOpc.get();
 633         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
 634         boolean interVpnLinkFound = false;
 635         for (InterVpnLink interVpnLink : interVpnLinks) {
 636             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 637             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 638                 && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 639             if (vpnIs1stEndpoint || vpnIs2ndEndpoint) {
 640                 interVpnLinkFound = true;
 641
 642                 Optional<InterVpnLinkState> vpnLinkState =
 643                     FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 644                 if (!vpnLinkState.isPresent()
 645                     || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active)) {
 646                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 647                         interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 648                         interVpnLink.getSecondEndpoint().getVpnUuid().getValue());
 649                     return;
 650                 }
 651
 652                 List<BigInteger> targetDpns = vpnIs1stEndpoint ? vpnLinkState.get().getFirstEndpointState().getDpId()
 653                     : vpnLinkState.get().getSecondEndpointState().getDpId();
 654                 Long lportTag = vpnIs1stEndpoint ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 655                     : vpnLinkState.get().getFirstEndpointState().getLportTag();
 656
 657                 LOG.trace("Installing flow in LFIB table for interVpnLink {}", interVpnLink.getName());
 658
 659                 for (BigInteger dpId : targetDpns) {
 660                     List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls());
 661
 662                     List<InstructionInfo> instructions =
 663                         Arrays.asList(new InstructionApplyActions(actionsInfos),
 664                             new InstructionWriteMetadata(
 665                                 MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 666                                     ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 667                                         NwConstants.L3VPN_SERVICE_INDEX)),
 668                                 MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 669                             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 670
 671                     LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for "
 672                         + "InterVpnLink {} in LFIB",
 673                         vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(),
 674                         dpId, interVpnLink.getName());
 675
 676                     makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY,
 677                         NwConstants.ADD_FLOW, null);
 678                 }
 679
 680                 break;
 681             }
 682         }
 683
 684         if (!interVpnLinkFound) {
 685             LOG.warn("VrfEntry=[prefix={} label={} nexthop={}] for VPN {} has origin INTERVPN but "
 686                 + "no InterVpnLink could be found",
 687                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
 688         }
 689     }
 690
 691
 692     /*
 693      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 694      */
 695     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 696                                                final VrfEntry vrfEntry, long vpnTag) {
 697         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 698         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 699             && vrfEntry.getNextHopAddressList().size() == 1);
 700         String destination = vrfEntry.getDestPrefix();
 701         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 702         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 703
 704         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 705         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 706         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 707         if (interVpnLink.getState().or(State.Error) != State.Active) {
 708             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 709                 destination, nextHop, interVpnLinkName);
 710             return;
 711         }
 712
 713         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 714         if (!optOtherEndpointLportTag.isPresent()) {
 715             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 716                 vpnUuid, interVpnLinkName);
 717             return;
 718         }
 719
 720         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 721         if (targetDpns.isEmpty()) {
 722             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 723                 vpnUuid, interVpnLinkName);
 724             return;
 725         }
 726
 727         String[] values = destination.split("/");
 728         String destPrefixIpAddress = values[0];
 729         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 730
 731         List<MatchInfo> matches = new ArrayList<>();
 732         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 733         matches.add(MatchEthernetType.IPV4);
 734
 735         if (prefixLength != 0) {
 736             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 737         }
 738
 739         List<Instruction> instructions =
 740             Arrays.asList(new InstructionWriteMetadata(
 741                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 742                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 743                             .L3VPN_SERVICE_INDEX)),
 744                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 745                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 746
 747         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 748         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 749         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 750             COOKIE_VM_FIB_TABLE, matches, instructions);
 751
 752         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 753             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 754
 755         for (BigInteger dpId : targetDpns) {
 756
 757             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nextHop={}] dpn {} for InterVpnLink {} in FIB",
 758                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(),
 759                 dpId, interVpnLink.getInterVpnLinkName());
 760
 761             mdsalManager.installFlow(dpId, flowEntity);
 762         }
 763     }
 764
 765
 766     // TODO Clean up the exception handling
 767     @SuppressWarnings("checkstyle:IllegalCatch")
 768     private <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 769                                                     InstanceIdentifier<T> path) {
 770
 771         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 772
 773         Optional<T> result = Optional.absent();
 774         try {
 775             result = tx.read(datastoreType, path).get();
 776         } catch (Exception e) {
 777             throw new RuntimeException(e);
 778         }
 779
 780         return result;
 781     }
 782
 783     private List<BigInteger> getDpnIdForPrefix(DataBroker broker, Long vpnId, String rd, VrfEntry vrfEntry) {
 784         List<BigInteger> returnLocalDpnId = new ArrayList<BigInteger>();
 785         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, vrfEntry.getDestPrefix());
 786
 787         if (localNextHopInfo == null) {
 788             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 789             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 790             if (extraRoute != null) {
 791                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 792                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 793                     if (nextHopIp != null) {
 794                         localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, nextHopIp + "/32");
 795                         if (localNextHopInfo != null) {
 796                             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 797                         }
 798                     }
 799                 }
 800             }
 801         } else {
 802             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 803         }
 804
 805         return returnLocalDpnId;
 806     }
 807
 808     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 809         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 810         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
 811         String localNextHopIP = vrfEntry.getDestPrefix();
 812
 813         if (localNextHopInfo == null) {
 814             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 815             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 816             if (extraRoute != null) {
 817                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 818                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 819                     if (nextHopIp != null) {
 820                         localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32");
 821                         localNextHopIP = nextHopIp + "/32";
 822                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 823                             rd, vrfEntry, vpnId);
 824                         returnLocalDpnId.add(dpnId);
 825                     }
 826                 }
 827             }
 828             if (localNextHopInfo == null) {
 829             /* imported routes case */
 830                 synchronized (vrfEntry.getLabel().toString().intern()) {
 831                     LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 832                     if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
 833                         && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 834                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 835                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 836                                 FibUtil.getVpnInstanceOpData(dataBroker, rd);
 837                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 838                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 839                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 840                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 841                                     localNextHopIP = lri.getPrefix();
 842                                 } else {
 843                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 844                                     localNextHopIP = lri.getPrefix();
 845                                 }
 846                             }
 847                             if (localNextHopInfo != null) {
 848                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 849                                     vrfEntry.getLabel(), localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 850                                 BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 851                                     vpnId, rd, vrfEntry, lri.getParentVpnid());
 852                                 returnLocalDpnId.add(dpnId);
 853                             }
 854                         }
 855                     }
 856                 }
 857             }
 858         } else {
 859             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 860             returnLocalDpnId.add(dpnId);
 861         }
 862
 863         return returnLocalDpnId;
 864     }
 865
 866     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 867                                                 final Long vpnId, final String rd,
 868                                                 final VrfEntry vrfEntry, Long parentVpnId) {
 869         if (localNextHopInfo != null) {
 870             final BigInteger dpnId = localNextHopInfo.getDpnId();
 871             if (!isVpnPresentInDpn(rd, dpnId)) {
 872                 LOG.error("The vpnName with vpnId {} rd {} is not available on dpn {}", vpnId, rd, dpnId.toString());
 873                 return BigInteger.ZERO;
 874             }
 875
 876             final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId,
 877                 localNextHopInfo.getVpnInterfaceName(), localNextHopIP, vrfEntry.getDestPrefix());
 878             if (groupId == 0) {
 879                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 880                     vrfEntry.getDestPrefix(), rd, localNextHopInfo.getVpnInterfaceName(), dpnId.toString());
 881                 return BigInteger.ZERO;
 882             }
 883             final List<InstructionInfo> instructions = Collections.singletonList(
 884                 new InstructionApplyActions(
 885                     Collections.singletonList(new ActionGroup(groupId))));
 886             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 887                 new InstructionApplyActions(
 888                     Arrays.asList(new ActionPopMpls(), new ActionGroup(groupId))));
 889             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 890                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 891                     dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
 892             } else {
 893                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. "
 894                     + "LFib and Terminating table entries will not be created.",
 895                     rd, vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpnId);
 896             }
 897             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 898             dataStoreCoordinator.enqueueJob("FIB-" + vpnId.toString()
 899                     + "-" + dpnId.toString() + "-" + vrfEntry.getDestPrefix(),
 900                 new Callable<List<ListenableFuture<Void>>>() {
 901                     @Override
 902                     public List<ListenableFuture<Void>> call() throws Exception {
 903                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 904                         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 905                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 906                             makeLFibTableEntry(dpnId, vrfEntry.getLabel(), lfibinstructions,
 907                                 DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 908                             makeTunnelTableEntry(dpnId, vrfEntry.getLabel(), groupId, tx);
 909                         }
 910                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 911                         futures.add(tx.submit());
 912                         return futures;
 913                     }
 914                 });
 915             return dpnId;
 916         }
 917         return BigInteger.ZERO;
 918     }
 919
 920     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId) {
 921         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 922         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 923         if (dpnInVpn.isPresent()) {
 924             return true;
 925         }
 926         return false;
 927     }
 928
 929     private LabelRouteInfo getLabelRouteInfo(Long label) {
 930         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 931             .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) label)).build();
 932         Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 933         if (opResult.isPresent()) {
 934             return opResult.get();
 935         }
 936         return null;
 937     }
 938
 939     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName) {
 940         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 941         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 942             .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) lri.getLabel())).build();
 943         if (lri == null) {
 944             return true;
 945         }
 946         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 947             ? lri.getVpnInstanceList() : new ArrayList<String>();
 948         if (vpnInstancesList.contains(vpnInstanceName)) {
 949             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 950             vpnInstancesList.remove(vpnInstanceName);
 951         }
 952         if (vpnInstancesList.size() == 0) {
 953             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 954             FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 955             return true;
 956         } else {
 957             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 958             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 959             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(),
 960                 FibUtil.DEFAULT_CALLBACK);
 961         }
 962         return false;
 963     }
 964
 965     private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 966                                       WriteTransaction tx) {
 967         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 968
 969         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 970
 971         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 972             dpId, label, groupId);
 973     }
 974
 975     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 976                                                 WriteTransaction tx) {
 977         List<MatchInfo> mkMatches = new ArrayList<>();
 978
 979         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 980             destDpId, label, actionsInfos);
 981
 982         // Matching metadata
 983         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 984         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 985
 986         List<InstructionInfo> mkInstructions = new ArrayList<>();
 987         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 988
 989         FlowEntity terminatingServiceTableFlowEntity =
 990             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 991             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
 992                 String.format("%s:%d", "TST Flow Entry ", label),
 993             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
 994
 995         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
 996
 997         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 998
 999         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1000         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1001             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1002             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1003             .child(Flow.class, flowKey).build();
1004         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(), true);
1005     }
1006
1007     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
1008         FlowEntity flowEntity;
1009         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1010         List<MatchInfo> mkMatches = new ArrayList<>();
1011         // Matching metadata
1012         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1013         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1014             NwConstants.INTERNAL_TUNNEL_TABLE,
1015             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1016             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1017             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1018         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
1019         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1020         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1021             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1022             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1023
1024         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1025         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1026     }
1027
1028     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1029         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1030         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1031         String localNextHopIP = vrfEntry.getDestPrefix();
1032
1033         if (localNextHopInfo == null) {
1034             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1035             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1036             if (extraRoute != null) {
1037                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1038                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1039                     if (nextHopIp != null) {
1040                         localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
1041                         localNextHopIP = nextHopIp + "/32";
1042                         BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1043                             vpnId, rd, vrfEntry, true /*isExtraRoute*/);
1044                         if (!dpnId.equals(BigInteger.ZERO)) {
1045                             returnLocalDpnId.add(dpnId);
1046                         }
1047                     }
1048                 }
1049             }
1050
1051             if (localNextHopInfo == null) {
1052               /* Imported VRF entry */
1053                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1054                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1055                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1056                     VpnNexthopBuilder vpnNexthopBuilder = new VpnNexthopBuilder();
1057                     vpnNexthopBuilder.setDpnId(lri.getDpnId());
1058                     BigInteger dpnId = checkDeleteLocalFibEntry(vpnNexthopBuilder.build(), localNextHopIP,
1059                         vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1060                     if (!dpnId.equals(BigInteger.ZERO)) {
1061                         returnLocalDpnId.add(dpnId);
1062                     }
1063                 }
1064             }
1065
1066         } else {
1067             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1068                 vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1069             if (!dpnId.equals(BigInteger.ZERO)) {
1070                 returnLocalDpnId.add(dpnId);
1071             }
1072         }
1073
1074         return returnLocalDpnId;
1075     }
1076
1077     private BigInteger checkDeleteLocalFibEntry(VpnNexthop localNextHopInfo, final String localNextHopIP,
1078                                                 final Long vpnId, final String rd,
1079                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
1080         if (localNextHopInfo != null) {
1081             final BigInteger dpnId = localNextHopInfo.getDpnId();
1082             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1083             dataStoreCoordinator.enqueueJob("FIB-" + vpnId.toString() + "-"
1084                     + dpnId.toString() + "-" + vrfEntry.getDestPrefix(),
1085                 new Callable<List<ListenableFuture<Void>>>() {
1086                     @Override
1087                     public List<ListenableFuture<Void>> call() throws Exception {
1088                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1089                         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
1090                             NwConstants.DEL_FLOW, tx);
1091                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1092                             makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null /* instructions */,
1093                                 DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1094                             removeTunnelTableEntry(dpnId, vrfEntry.getLabel(), tx);
1095                         }
1096                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1097                         futures.add(tx.submit());
1098                         return futures;
1099                     }
1100                 });
1101             //TODO: verify below adjacency call need to be optimized (?)
1102             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1103             return dpnId;
1104         }
1105         return BigInteger.ZERO;
1106     }
1107
1108     private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
1109         return InstanceIdentifier.builder(VpnToExtraroute.class)
1110             .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
1111                 new ExtrarouteKey(ipPrefix)).build();
1112     }
1113
1114     private Extraroute getVpnToExtraroute(String rd, String ipPrefix) {
1115         Optional<Extraroute> extraRouteInfo =
1116             FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getVpnToExtrarouteIdentifier(rd, ipPrefix));
1117         return extraRouteInfo.isPresent() ? extraRouteInfo.get() : null;
1118
1119     }
1120
1121     private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
1122         try {
1123             Future<RpcResult<GetTunnelTypeOutput>> result = interfaceManager.getTunnelType(
1124                 new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
1125             RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
1126             if (!rpcResult.isSuccessful()) {
1127                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
1128             } else {
1129                 return rpcResult.getResult().getTunnelType();
1130             }
1131
1132         } catch (InterruptedException | ExecutionException e) {
1133             LOG.warn("Exception when getting tunnel interface Id for tunnel type", e);
1134         }
1135
1136         return null;
1137
1138     }
1139
1140     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1141                                       final VrfEntry vrfEntry, WriteTransaction tx) {
1142         Boolean wrTxPresent = true;
1143         if (tx == null) {
1144             wrTxPresent = false;
1145             tx = dataBroker.newWriteOnlyTransaction();
1146         }
1147         String rd = vrfTableKey.getRouteDistinguisher();
1148         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1149             vrfEntry.getDestPrefix(), rd, remoteDpnId);
1150
1151         List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1152         if (adjacencyResults.isEmpty()) {
1153             LOG.error("Could not get interface for nexthop: {} in vpn {}",
1154                 vrfEntry.getNextHopAddressList(), rd);
1155             LOG.warn("Failed to add Route: {} in vpn: {}",
1156                 vrfEntry.getDestPrefix(), rd);
1157             return;
1158         }
1159
1160         for (AdjacencyResult adjacencyResult : adjacencyResults) {
1161             List<ActionInfo> actionInfos = new ArrayList<>();
1162             String egressInterface = adjacencyResult.getInterfaceName();
1163             if (Tunnel.class.equals(adjacencyResult.getInterfaceType())) {
1164                 addTunnelInterfaceActions(egressInterface, vpnId, vrfEntry, actionInfos);
1165             } else {
1166                 addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1167             }
1168             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(egressInterface);
1169             if (egressActions.isEmpty()) {
1170                 LOG.error("Failed to retrieve egress action for prefix {} nextHop {} interface {}. "
1171                     + "Aborting remote FIB entry creation.",
1172                     vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), egressInterface);
1173                 return;
1174             }
1175             actionInfos.addAll(egressActions);
1176             List<InstructionInfo> instructions = new ArrayList<>();
1177             instructions.add(new InstructionApplyActions(actionInfos));
1178             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1179         }
1180         if (!wrTxPresent) {
1181             tx.submit();
1182         }
1183         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1184     }
1185
1186     private void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, List<ActionInfo> actionInfos) {
1187         String ipPrefix = vrfEntry.getDestPrefix();
1188         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, ipPrefix);
1189         if (prefixInfo == null) {
1190             LOG.debug("No prefix info found for prefix {}", ipPrefix);
1191             return;
1192         }
1193
1194         String ifName = prefixInfo.getVpnInterfaceName();
1195         if (ifName == null) {
1196             LOG.warn("Failed to get VPN interface for prefix {}", ipPrefix);
1197             return;
1198         }
1199
1200         String macAddress = FibUtil.getMacAddressFromPrefix(dataBroker, ifName, ipPrefix);
1201         if (macAddress == null) {
1202             LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix);
1203             return;
1204         }
1205
1206         actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(), new MacAddress(macAddress)));
1207     }
1208
1209     private void addTunnelInterfaceActions(String tunnelInterface, long vpnId, VrfEntry vrfEntry,
1210                                            List<ActionInfo> actionInfos) {
1211         Class<? extends TunnelTypeBase> tunnelType = getTunnelType(tunnelInterface);
1212         if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
1213             LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1214             actionInfos.add(new ActionPushMpls());
1215             actionInfos.add(new ActionSetFieldMplsLabel(vrfEntry.getLabel()));
1216             actionInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
1217         } else {
1218             int label = vrfEntry.getLabel().intValue();
1219             BigInteger tunnelId;
1220             // FIXME vxlan vni bit set is not working properly with OVS.need to
1221             // revisit
1222             if (tunnelType.equals(TunnelTypeVxlan.class)) {
1223                 tunnelId = BigInteger.valueOf(label);
1224             } else {
1225                 tunnelId = BigInteger.valueOf(label);
1226             }
1227
1228             LOG.debug("adding set tunnel id action for label {}", label);
1229             actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
1230             addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1231         }
1232     }
1233
1234     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1235         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1236         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1237         if (dpnInVpn.isPresent()) {
1238             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1239             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1240
1241             if (vpnInterfaces.remove(currVpnInterface)) {
1242                 if (vpnInterfaces.isEmpty()) {
1243                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1244                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1245                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
1246                 } else {
1247                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1248                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1249                         VpnInterfaces.class,
1250                         new VpnInterfacesKey(intfName)));
1251                 }
1252             }
1253         }
1254     }
1255
1256     private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
1257     /* Get interface info from prefix to interface mapping;
1258         Use the interface info to get the corresponding vpn interface op DS entry,
1259         remove the adjacency corresponding to this fib entry.
1260         If adjacency removed is the last adjacency, clean up the following:
1261          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1262          - prefix to interface entry
1263          - vpn interface op DS
1264      */
1265         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1266         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1267         Extraroute extraRoute = null;
1268         if (prefixInfo == null) {
1269             extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1270             if (extraRoute != null) {
1271                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1272                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1273
1274                     if (nextHopIp != null) {
1275                         prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32");
1276                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1277                     }
1278                 }
1279             }
1280             if (prefixInfo == null) {
1281                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1282                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1283                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1284                     PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1285                     prefixBuilder.setDpnId(lri.getDpnId());
1286                     prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1287                     prefixBuilder.setIpAddress(lri.getPrefix());
1288                     prefixInfo = prefixBuilder.build();
1289                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1290                         vrfEntry.getLabel(), prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1291                     checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1292                 }
1293             }
1294         } else {
1295             checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1296         }
1297     }
1298
1299     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1300                                           final VrfEntry vrfEntry, final Extraroute extraRoute) {
1301
1302         if (prefixInfo == null) {
1303             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1304             return; //Don't have any info for this prefix (shouldn't happen); need to return
1305         }
1306
1307         String ifName = prefixInfo.getVpnInterfaceName();
1308         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1309         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1310             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1311     }
1312
1313     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1314         Prefixes prefixInfo;
1315         Long vpnId;
1316         String rd;
1317         VrfEntry vrfEntry;
1318         Extraroute extraRoute;
1319
1320         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1321                                          final VrfEntry vrfEntry, final Extraroute extraRoute) {
1322             this.prefixInfo = prefixInfo;
1323             this.vpnId = vpnId;
1324             this.rd = rd;
1325             this.vrfEntry = vrfEntry;
1326             this.extraRoute = extraRoute;
1327         }
1328
1329         @Override
1330         public List<ListenableFuture<Void>> call() throws Exception {
1331             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1332             // to call the respective helpers.
1333
1334             //First Cleanup LabelRouteInfo
1335             synchronized (vrfEntry.getLabel().toString().intern()) {
1336                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1337                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1338                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1339                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1340                         FibUtil.getVpnInstanceOpData(dataBroker, rd);
1341                     String vpnInstanceName = "";
1342                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1343                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1344                     }
1345                     boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName);
1346                     if (lriRemoved) {
1347                         String parentRd = lri.getParentVpnRd();
1348                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1349                             FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1350                     }
1351                 } else {
1352                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1353                         FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1354                 }
1355             }
1356             String ifName = prefixInfo.getVpnInterfaceName();
1357             Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1358                 FibUtil.getVpnInterfaceIdentifier(ifName));
1359             if (optvpnInterface.isPresent()) {
1360                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1361                 if (vpnId != associatedVpnId) {
1362                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1363                         vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1364                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1365                     return null;
1366                 } else {
1367                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1368                         vrfEntry.getDestPrefix(), associatedVpnId);
1369                 }
1370             }
1371             if (extraRoute != null) {
1372                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1373                     FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1374             }
1375             Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1376                 FibUtil.getAdjListPath(ifName));
1377             int numAdj = 0;
1378             if (optAdjacencies.isPresent()) {
1379                 numAdj = optAdjacencies.get().getAdjacency().size();
1380             }
1381             //remove adjacency corr to prefix
1382             if (numAdj > 1) {
1383                 LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
1384                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1385                     FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1386             }
1387             if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
1388                 //clean up the vpn interface from DpnToVpn list
1389                 LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1390                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1391                     FibUtil.getVpnInterfaceIdentifier(ifName));
1392             }
1393             return null;
1394         }
1395     }
1396
1397     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1398         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1399         final String rd = vrfTableKey.getRouteDistinguisher();
1400         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1401         if (vpnInstance == null) {
1402             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1403             return;
1404         }
1405         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1406         long elanTag = 0L;
1407         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1408         if (subnetRoute != null) {
1409             elanTag = subnetRoute.getElantag();
1410             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1411                 rd, vrfEntry.getDestPrefix(), elanTag);
1412             if (vpnToDpnList != null) {
1413                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1414                 dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
1415                     new Callable<List<ListenableFuture<Void>>>() {
1416                         @Override
1417                         public List<ListenableFuture<Void>> call() throws Exception {
1418                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1419
1420                             for (final VpnToDpnList curDpn : vpnToDpnList) {
1421
1422                                 makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1423                                     vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1424                                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1425                                     makeLFibTableEntry(curDpn.getDpnId(), vrfEntry.getLabel(), null,
1426                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1427                                 }
1428                             }
1429                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1430                             futures.add(tx.submit());
1431                             return futures;
1432                         }
1433                     });
1434             }
1435             synchronized (vrfEntry.getLabel().toString().intern()) {
1436                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1437                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1438                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1439                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1440                         FibUtil.getVpnInstanceOpData(dataBroker, rd);
1441                     String vpnInstanceName = "";
1442                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1443                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1444                     }
1445                     boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName);
1446                     if (lriRemoved) {
1447                         String parentRd = lri.getParentVpnRd();
1448                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1449                             FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1450                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as "
1451                             + "labelRouteInfo cleared", vrfEntry.getLabel(), rd,
1452                             vrfEntry.getDestPrefix());
1453                     }
1454                 } else {
1455                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1456                         FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1457                     LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1458                         vrfEntry.getLabel(), rd, vrfEntry.getDestPrefix());
1459                 }
1460             }
1461             return;
1462         }
1463         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnInstance.getVpnId(), NwConstants.DEL_FLOW)) {
1464             return;
1465         }
1466
1467         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1468             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1469         if (vpnToDpnList != null) {
1470             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1471             dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
1472                 new Callable<List<ListenableFuture<Void>>>() {
1473                     @Override
1474                     public List<ListenableFuture<Void>> call() throws Exception {
1475                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1476
1477                         if (localDpnIdList.size() <= 0) {
1478                             for (VpnToDpnList curDpn : vpnToDpnList) {
1479                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1480                                     if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1481                                         deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1482                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1483                                     }
1484                                 } else {
1485                                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1486                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1487                                 }
1488                             }
1489                         } else {
1490                             for (BigInteger localDpnId : localDpnIdList) {
1491                                 for (VpnToDpnList curDpn : vpnToDpnList) {
1492                                     if (!curDpn.getDpnId().equals(localDpnId)) {
1493                                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1494                                             if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1495                                                 deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1496                                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1497                                             }
1498                                         } else {
1499                                             deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1500                                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1501                                         }
1502                                     }
1503                                 }
1504                             }
1505                         }
1506                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1507                         futures.add(tx.submit());
1508                         return futures;
1509                     }
1510                 });
1511         }
1512
1513         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1514         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1515         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1516
1517         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1518         // of the interVpnLink.
1519         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(this.dataBroker, rd);
1520         if (optVpnUuid.isPresent()) {
1521             String vpnUuid = optVpnUuid.get();
1522             List<String> routeNexthoplist = vrfEntry.getNextHopAddressList();
1523             if (routeNexthoplist.isEmpty()) {
1524                 LOG.trace("NextHopList is empty for VrfEntry {}", vrfEntry);
1525                 return;
1526             }
1527             String routeNexthop = routeNexthoplist.get(0);
1528             Optional<InterVpnLinkDataComposite> optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1529             if (optInterVpnLink.isPresent()) {
1530                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1531                 if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1532                     // This is route that points to the other endpoint of an InterVpnLink
1533                     // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
1534                     removeInterVPNLinkRouteFlows(interVpnLink.getInterVpnLinkName(),
1535                         interVpnLink.isFirstEndpointVpnName(rd),
1536                         vrfEntry);
1537                 }
1538             }
1539         }
1540
1541     }
1542
1543     /*
1544       Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1545       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1546       provided by ResourceBatchingManager
1547      */
1548     private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
1549                                   final VrfEntry vrfEntry) {
1550         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1551
1552         final String rd = vrfTableKey.getRouteDistinguisher();
1553         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1554         if (vpnInstance == null) {
1555             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1556             return;
1557         }
1558         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1559         if (vpnToDpnList != null) {
1560             for (VpnToDpnList curDpn : vpnToDpnList) {
1561                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1562                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(),
1563                         vrfTableKey, vrfEntry, writeTx);
1564                 }
1565             }
1566         }
1567     }
1568
1569     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1570                                   final long vpnId, final VrfTablesKey vrfTableKey,
1571                                   final VrfEntry vrfEntry, WriteTransaction tx) {
1572
1573         Boolean wrTxPresent = true;
1574         if (tx == null) {
1575             wrTxPresent = false;
1576             tx = dataBroker.newWriteOnlyTransaction();
1577         }
1578
1579         LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
1580             vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
1581         String rd = vrfTableKey.getRouteDistinguisher();
1582
1583         if (localDpnId != null) {
1584             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1585             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1586             return;
1587         }
1588
1589         // below two reads are kept as is, until best way is found to identify dpnID
1590         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1591         Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1592
1593         if (localNextHopInfo == null && extraRoute != null) {
1594             // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1595             for (String nextHopIp : extraRoute.getNexthopIpList()) {
1596                 localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
1597                 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1598             }
1599         } else {
1600             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1601         }
1602         if (!wrTxPresent) {
1603             tx.submit();
1604         }
1605     }
1606
1607     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1608                                            VrfEntry vrfEntry, String rd, WriteTransaction tx) {
1609         boolean isRemoteRoute = true;
1610         if (localNextHopInfo != null) {
1611             isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId());
1612         }
1613         if (isRemoteRoute) {
1614             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1615             return true;
1616         } else {
1617             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}",
1618                 rd, vrfEntry.getDestPrefix(), remoteDpnId);
1619             return false;
1620         }
1621     }
1622
1623     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry, String rd, WriteTransaction tx) {
1624         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1625         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1626     }
1627
1628     private long get(byte[] rawIpAddress) {
1629         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1630             + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1631     }
1632
1633     private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1634                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1635         Boolean wrTxPresent = true;
1636         if (tx == null) {
1637             wrTxPresent = false;
1638             tx = dataBroker.newWriteOnlyTransaction();
1639         }
1640
1641         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1642         String[] values = vrfEntry.getDestPrefix().split("/");
1643         String ipAddress = values[0];
1644         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1645         if (addOrRemove == NwConstants.ADD_FLOW) {
1646             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1647         } else {
1648             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1649         }
1650         InetAddress destPrefix;
1651         try {
1652             destPrefix = InetAddress.getByName(ipAddress);
1653         } catch (UnknownHostException e) {
1654             LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1655             return;
1656         }
1657
1658         List<MatchInfo> matches = new ArrayList<>();
1659
1660         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
1661
1662         if (destPrefix instanceof Inet4Address) {
1663             matches.add(MatchEthernetType.IPV4);
1664             if (prefixLength != 0) {
1665                 matches.add(new MatchIpv4Destination(destPrefix.getHostAddress(), Integer.toString(prefixLength)));
1666             }
1667         } else {
1668             matches.add(MatchEthernetType.IPV6);
1669             if (prefixLength != 0) {
1670                 matches.add(new MatchIpv6Destination(destPrefix.getHostAddress() + "/" + prefixLength));
1671             }
1672         }
1673
1674         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1675         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1676         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority,
1677             flowRef, 0, 0,
1678             COOKIE_VM_FIB_TABLE, matches, instructions);
1679
1680         Flow flow = flowEntity.getFlowBuilder().build();
1681         String flowId = flowEntity.getFlowId();
1682         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1683         Node nodeDpn = buildDpnNode(dpId);
1684
1685         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1686             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1687             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1688
1689         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1690             SubTransaction subTransaction = new SubTransactionImpl();
1691             if (addOrRemove == NwConstants.ADD_FLOW) {
1692                 subTransaction.setInstanceIdentifier(flowInstanceId);
1693                 subTransaction.setInstance(flow);
1694                 subTransaction.setAction(SubTransaction.CREATE);
1695             } else {
1696                 subTransaction.setInstanceIdentifier(flowInstanceId);
1697                 subTransaction.setAction(SubTransaction.DELETE);
1698             }
1699             transactionObjects.add(subTransaction);
1700         }
1701
1702         if (addOrRemove == NwConstants.ADD_FLOW) {
1703             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
1704         } else {
1705             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1706         }
1707
1708         if (!wrTxPresent) {
1709             tx.submit();
1710         }
1711     }
1712
1713     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1714     private Node buildDpnNode(BigInteger dpnId) {
1715         NodeId nodeId = new NodeId("openflow:" + dpnId);
1716         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1717
1718         return nodeDpn;
1719     }
1720
1721     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1722                                     int addOrRemove, WriteTransaction tx) {
1723         Boolean wrTxPresent = true;
1724         if (tx == null) {
1725             wrTxPresent = false;
1726             tx = dataBroker.newWriteOnlyTransaction();
1727         }
1728
1729         List<MatchInfo> matches = new ArrayList<MatchInfo>();
1730         matches.add(MatchEthernetType.MPLS_UNICAST);
1731         matches.add(new MatchMplsLabel(label));
1732
1733         // Install the flow entry in L3_LFIB_TABLE
1734         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1735
1736         FlowEntity flowEntity;
1737         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1738             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1739         Flow flow = flowEntity.getFlowBuilder().build();
1740         String flowId = flowEntity.getFlowId();
1741         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1742         Node nodeDpn = buildDpnNode(dpId);
1743         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1744             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1745             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1746
1747         if (addOrRemove == NwConstants.ADD_FLOW) {
1748             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
1749         } else {
1750             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1751         }
1752         if (!wrTxPresent) {
1753             tx.submit();
1754         }
1755
1756         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1757             dpId, label, instructions, flowKey, (NwConstants.ADD_FLOW == addOrRemove) ? "ADDED" : "REMOVED");
1758     }
1759
1760     private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress,
1761                                       final String ipPrefixAddress) {
1762         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}", dpId, vpnId, ipAddress);
1763         try {
1764             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress);
1765         } catch (NullPointerException e) {
1766             LOG.trace("", e);
1767         }
1768     }
1769
1770     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1771                                     final FutureCallback<List<Void>> callback) {
1772         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1773         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1774         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1775         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1776         if (!vrfTable.isPresent()) {
1777             LOG.warn("VRF Table not yet available for RD {}", rd);
1778             if (callback != null) {
1779                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1780                 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1781                 Futures.addCallback(listenableFuture, callback);
1782             }
1783             return;
1784         }
1785         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1786         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1787             new Callable<List<ListenableFuture<Void>>>() {
1788                 @Override
1789                 public List<ListenableFuture<Void>> call() throws Exception {
1790                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1791                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1792                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1793                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1794                             SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1795                             if (subnetRoute != null) {
1796                                 long elanTag = subnetRoute.getElantag();
1797                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1798                                 continue;
1799                             }
1800                             RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1801                             if (routerInt != null) {
1802                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1803                                     rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1804                                 installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
1805                                     routerInt.getIpAddress(),
1806                                     new MacAddress(routerInt.getMacAddress()), NwConstants.ADD_FLOW);
1807                                 continue;
1808                             }
1809                             //Handle local flow creation for imports
1810                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1811                                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1812                                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1813                                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1814                                     if (lri.getDpnId().equals(dpnId)) {
1815                                         createLocalFibEntry(vpnId, rd, vrfEntry);
1816                                         continue;
1817                                     }
1818                                 }
1819                             }
1820                             // Passing null as we don't know the dpn
1821                             // to which prefix is attached at this point
1822                             createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1823                         }
1824                         //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1825                         futures.add(tx.submit());
1826                     }
1827                     if (callback != null) {
1828                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1829                         Futures.addCallback(listenableFuture, callback);
1830                     }
1831                     return futures;
1832                 }
1833             });
1834     }
1835
1836
1837     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1838                                             final String localNextHopIp, final String remoteNextHopIp) {
1839         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1840             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1841         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1842         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1843         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1844         if (vrfTable.isPresent()) {
1845             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1846             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1847                 new Callable<List<ListenableFuture<Void>>>() {
1848                     @Override
1849                     public List<ListenableFuture<Void>> call() throws Exception {
1850                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1851                         synchronized (vpnInstance.getVpnInstanceName().intern()) {
1852                             WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1853                             for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1854                                 if (!vrfEntry.getNextHopAddressList().isEmpty()) {
1855                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1856                                         if (remoteNextHopIp.trim()
1857                                             .equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1858                                             LOG.trace(" creating remote FIB entry for prefix {} rd {}",
1859                                                 vrfEntry.getDestPrefix(), rd);
1860                                             createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(),
1861                                                 vrfEntry, writeCfgTxn);
1862                                         }
1863                                     }
1864                                 }
1865                             }
1866                             futures.add(writeCfgTxn.submit());
1867                         }
1868                         return futures;
1869                     }
1870                 });
1871         }
1872     }
1873
1874     public void populateInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1875                                             final String localNextHopIp, final String remoteNextHopIp) {
1876         LOG.trace("populateInternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1877             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1878         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1879         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1880         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1881         if (vrfTable.isPresent()) {
1882             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1883             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1884                 new Callable<List<ListenableFuture<Void>>>() {
1885                     @Override
1886                     public List<ListenableFuture<Void>> call() throws Exception {
1887                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1888                         synchronized (vpnInstance.getVpnInstanceName().intern()) {
1889                             WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1890                             for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1891                                 // Handle Internal Routes only (i.e., STATIC for now)
1892                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC) {
1893                                     SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1894                                         /* Ignore SubnetRoute entry */
1895                                     if (subnetRoute == null) {
1896                                         if (!vrfEntry.getNextHopAddressList().isEmpty()) {
1897                                             if (remoteNextHopIp.trim()
1898                                                 .equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1899                                                 LOG.trace(" creating remote FIB entry for prefix {} rd {} on Dpn {}",
1900                                                     vrfEntry.getDestPrefix(), rd, dpnId);
1901                                                 createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry,
1902                                                     writeCfgTxn);
1903                                             }
1904                                         }
1905                                     }
1906                                 }
1907                             }
1908                             futures.add(writeCfgTxn.submit());
1909                         }
1910                         return futures;
1911                     }
1912                 });
1913         }
1914     }
1915
1916     public void manageRemoteRouteOnDPN(final boolean action,
1917                                        final BigInteger localDpnId,
1918                                        final long vpnId,
1919                                        final String rd,
1920                                        final String destPrefix,
1921                                        final String destTepIp) {
1922         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1923
1924         if (vpnInstance == null) {
1925             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1926             return;
1927         }
1928         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1929         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + localDpnId.toString(),
1930             new Callable<List<ListenableFuture<Void>>>() {
1931                 @Override
1932                 public List<ListenableFuture<Void>> call() throws Exception {
1933                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1934                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1935                         WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1936                         VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1937                         VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1938                         if (vrfEntry == null) {
1939                             return futures;
1940                         }
1941                         LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1942                             action, localDpnId, vpnId, rd, destPrefix);
1943                         List<String> nhList = new ArrayList<String>();
1944                         List<String> nextHopAddressList = vrfEntry.getNextHopAddressList();
1945                         VrfEntry modVrfEntry;
1946                         if (nextHopAddressList == null || (nextHopAddressList.isEmpty())) {
1947                             nhList = Arrays.asList(destTepIp);
1948                             modVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(nhList).build();
1949                         } else {
1950                             modVrfEntry = vrfEntry;
1951                         }
1952
1953                         if (action == true) {
1954                             LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1955                             createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
1956                         } else {
1957                             LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1958                             deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
1959                         }
1960                         futures.add(writeTransaction.submit());
1961                     }
1962                     return futures;
1963                 }
1964             });
1965     }
1966
1967     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1968                                  final FutureCallback<List<Void>> callback) {
1969         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1970         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1971         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1972         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1973         if (vrfTable.isPresent()) {
1974             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1975             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1976                 new Callable<List<ListenableFuture<Void>>>() {
1977                     @Override
1978                     public List<ListenableFuture<Void>> call() throws Exception {
1979                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1980                         synchronized (vpnInstance.getVpnInstanceName().intern()) {
1981                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1982                             for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1983                                     /* Handle subnet routes here */
1984                                 SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1985                                 if (subnetRoute != null) {
1986                                     LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn",
1987                                         vrfEntry.getDestPrefix(),
1988                                         dpnId, rd);
1989                                     makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1990                                     makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY,
1991                                         NwConstants.DEL_FLOW, tx);
1992                                     LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}",
1993                                         vrfEntry.getLabel(), rd,
1994                                         vrfEntry.getDestPrefix());
1995                                     continue;
1996                                 }
1997                                 // ping responder for router interfaces
1998                                 RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1999                                 if (routerInt != null) {
2000                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
2001                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
2002                                     installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
2003                                         routerInt.getIpAddress(),
2004                                         new MacAddress(routerInt.getMacAddress()), NwConstants.DEL_FLOW);
2005                                     continue;
2006                                 }
2007                                 // Passing null as we don't know the dpn
2008                                 // to which prefix is attached at this point
2009                                 deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
2010                             }
2011                             futures.add(tx.submit());
2012                             if (callback != null) {
2013                                 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
2014                                 Futures.addCallback(listenableFuture, callback);
2015                             }
2016                         }
2017                         return futures;
2018                     }
2019                 });
2020         }
2021     }
2022
2023     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
2024                                            final String localNextHopIp, final String remoteNextHopIp) {
2025         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
2026                 + " localNexthopIp {} , remoteNexhtHopIp {}",
2027             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2028         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2029         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2030         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2031         if (vrfTable.isPresent()) {
2032             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2033             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2034                 new Callable<List<ListenableFuture<Void>>>() {
2035                     @Override
2036                     public List<ListenableFuture<Void>> call() throws Exception {
2037                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2038                         synchronized (vpnInstance.getVpnInstanceName().intern()) {
2039                             WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
2040                             for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
2041                                 if (!vrfEntry.getNextHopAddressList().isEmpty()) {
2042                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2043                                         if (remoteNextHopIp.trim()
2044                                             .equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
2045                                             LOG.trace(" deleting remote FIB entry {}", vrfEntry);
2046                                             deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(),
2047                                                 vrfEntry, writeTransaction);
2048                                         }
2049                                     }
2050                                 }
2051                             }
2052                             futures.add(writeTransaction.submit());
2053                         }
2054                         return futures;
2055                     }
2056                 });
2057
2058         }
2059     }
2060
2061     public void cleanUpInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
2062                                            final String localNextHopIp, final String remoteNextHopIp) {
2063         LOG.trace("cleanUpInternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
2064                 + " localNexthopIp {} , remoteNexhtHopIp {}",
2065             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2066         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2067         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2068         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2069         if (vrfTable.isPresent()) {
2070             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2071             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2072                 new Callable<List<ListenableFuture<Void>>>() {
2073                     @Override
2074                     public List<ListenableFuture<Void>> call() throws Exception {
2075                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2076                         synchronized (vpnInstance.getVpnInstanceName().intern()) {
2077                             WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
2078                             for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
2079                                 // Handle Internal Routes only (i.e, STATIC for now)
2080                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC) {
2081                                     SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
2082                                         /* Ignore SubnetRoute entry */
2083                                     if (subnetRoute == null) {
2084                                         if (!vrfEntry.getNextHopAddressList().isEmpty()) {
2085                                             if (remoteNextHopIp.trim()
2086                                                 .equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
2087                                                 LOG.trace(" deleting remote FIB entry {}", vrfEntry);
2088                                                 deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(),
2089                                                     vrfEntry, writeTransaction);
2090                                             }
2091                                         }
2092                                     }
2093                                 }
2094                             }
2095                             futures.add(writeTransaction.submit());
2096                         }
2097                         return futures;
2098                     }
2099                 });
2100         }
2101     }
2102
2103     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
2104         InstanceIdentifierBuilder<VrfTables> idBuilder =
2105             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
2106         InstanceIdentifier<VrfTables> id = idBuilder.build();
2107         return id;
2108     }
2109
2110     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
2111         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2112             .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(label).append(NwConstants.FLOWID_SEPARATOR)
2113             .append(priority).toString();
2114     }
2115
2116     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
2117         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2118             .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
2119             .append(rd).append(NwConstants.FLOWID_SEPARATOR)
2120             .append(priority).append(NwConstants.FLOWID_SEPARATOR)
2121             .append(destPrefix.getHostAddress()).toString();
2122     }
2123
2124     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
2125         return new StringBuilder(64).append(FLOWID_PREFIX)
2126             .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
2127             .append(prefix).append(NwConstants.FLOWID_SEPARATOR)
2128             .append(nextHop).toString();
2129     }
2130
2131     protected List<AdjacencyResult> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
2132                                                      final VrfEntry vrfEntry, String rd) {
2133         List<AdjacencyResult> adjacencyList = new ArrayList<>();
2134         List<String> prefixIpList = new ArrayList<>();
2135         LOG.trace("resolveAdjacency called with remotedDpnId {}, vpnId{}, VrfEntry {}",
2136             remoteDpnId, vpnId, vrfEntry);
2137         try {
2138             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2139                 Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
2140                 if (extraRoute == null) {
2141                     prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
2142                 } else {
2143                     prefixIpList = new ArrayList<>();
2144                     for (String extraRouteIp : extraRoute.getNexthopIpList()) {
2145                         prefixIpList.add(extraRouteIp + "/32");
2146                     }
2147                 }
2148             } else {
2149                 prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
2150             }
2151
2152             for (String prefixIp : prefixIpList) {
2153                 for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
2154                     LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
2155                     AdjacencyResult adjacencyResult = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
2156                         prefixIp, nextHopIp);
2157                     if (adjacencyResult != null && !adjacencyList.contains(adjacencyResult)) {
2158                         adjacencyList.add(adjacencyResult);
2159                     }
2160                 }
2161             }
2162         } catch (NullPointerException e) {
2163             LOG.trace("", e);
2164         }
2165         return adjacencyList;
2166     }
2167
2168     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
2169         InstanceIdentifier<VpnInstanceOpDataEntry> id =
2170             InstanceIdentifier.create(VpnInstanceOpData.class)
2171                 .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
2172         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
2173             FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
2174         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
2175     }
2176
2177     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2178         return new StringBuffer().append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2179             .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(tableMiss)
2180             .append(FLOWID_PREFIX).toString();
2181     }
2182
2183     /*
2184      * Install flow entry in protocol table to forward mpls
2185      * coming through gre tunnel to LFIB table.
2186      */
2187     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
2188         final BigInteger cookieProtocolTable = new BigInteger("1070000", 16);
2189         // Instruction to goto L3 InterfaceTable
2190         List<InstructionInfo> instructions = new ArrayList<>();
2191         instructions.add(new InstructionGotoTable(NwConstants.L3_LFIB_TABLE));
2192         List<MatchInfo> matches = new ArrayList<MatchInfo>();
2193         matches.add(MatchEthernetType.MPLS_UNICAST);
2194         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2195             getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2196                 NwConstants.L3_LFIB_TABLE),
2197             DEFAULT_FIB_FLOW_PRIORITY,
2198             "Protocol Table For LFIB",
2199             0, 0,
2200             cookieProtocolTable,
2201             matches, instructions);
2202
2203         if (addOrRemove == NwConstants.ADD_FLOW) {
2204             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2205             mdsalManager.installFlow(flowEntityToLfib);
2206         } else {
2207             mdsalManager.removeFlow(flowEntityToLfib);
2208         }
2209     }
2210
2211     public List<String> printFibEntries() {
2212         List<String> result = new ArrayList<>();
2213         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2214         result.add("-------------------------------------------------------------------");
2215         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2216         Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2217         if (fibEntries.isPresent()) {
2218             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2219             for (VrfTables vrfTable : vrfTables) {
2220                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2221                     for (String nextHop : vrfEntry.getNextHopAddressList()) {
2222                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2223                             vrfTable.getRouteDistinguisher(),
2224                             vrfEntry.getDestPrefix(), nextHop, vrfEntry.getLabel(), vrfEntry.getOrigin()));
2225                     }
2226                     if (vrfEntry.getNextHopAddressList().isEmpty()) {
2227                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2228                             vrfTable.getRouteDistinguisher(),
2229                             vrfEntry.getDestPrefix(), "local", vrfEntry.getLabel(), vrfEntry.getOrigin()));
2230                     }
2231                 }
2232             }
2233         }
2234         return result;
2235     }
2236
2237
2238     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2239         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2240             .child(VrfTables.class, new VrfTablesKey(rd))
2241             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2242         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2243         if (vrfEntry.isPresent()) {
2244             return vrfEntry.get();
2245         }
2246         return null;
2247     }
2248
2249     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2250         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2251             .child(VrfTables.class, new VrfTablesKey(rd))
2252             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2253         return vrfEntryId;
2254     }
2255
2256     protected Boolean isIpv4Address(String ipAddress) {
2257         try {
2258             InetAddress address = InetAddress.getByName(ipAddress);
2259             if (address instanceof Inet4Address) {
2260                 return true;
2261             }
2262         } catch (UnknownHostException e) {
2263             LOG.warn("Invalid ip address {}", ipAddress, e);
2264             return false;
2265         }
2266         return false;
2267     }
2268
2269     protected Boolean installRouterFibEntries(final VrfEntry vrfEntry, final Collection<VpnToDpnList> vpnToDpnList,
2270                                               long vpnId, int addOrRemove) {
2271         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
2272         if (routerInt != null && vpnToDpnList != null) {
2273             String routerId = routerInt.getUuid();
2274             String macAddress = routerInt.getMacAddress();
2275             String ipValue = routerInt.getIpAddress();
2276             LOG.trace("createFibEntries - Router augmented vrfentry found for for router uuid:{}, ip:{}, mac:{}",
2277                 routerId, ipValue, macAddress);
2278             for (VpnToDpnList vpnDpn : vpnToDpnList) {
2279                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
2280                     installRouterFibEntry(vrfEntry, vpnDpn.getDpnId(), vpnId, routerId, ipValue,
2281                         new MacAddress(macAddress), addOrRemove);
2282                 }
2283             }
2284             return true;
2285         }
2286         return false;
2287     }
2288
2289     public void installRouterFibEntry(final VrfEntry vrfEntry, BigInteger dpnId, long vpnId, String routerUuid,
2290                                       String routerInternalIp, MacAddress routerMac, int addOrRemove) {
2291         String[] subSplit = routerInternalIp.split("/");
2292         if (!isIpv4Address(subSplit[0])) {
2293             // Ping responder using OpenFlow rules is only supported for IPv4, hence skipping.
2294             return;
2295         }
2296
2297         String addRemoveStr = (addOrRemove == NwConstants.ADD_FLOW) ? "ADD_FLOW" : "DELETE_FLOW";
2298         LOG.trace("{}: bulding Echo Flow entity for dpid:{}, router_ip:{}, vpnId:{}, subSplit:{} ", addRemoveStr,
2299             dpnId, routerInternalIp, vpnId, subSplit[0]);
2300
2301         List<MatchInfo> matches = new ArrayList<>();
2302
2303         matches.add(MatchIpProtocol.ICMP);
2304         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
2305         matches.add(new MatchIcmpv4((short) 8, (short) 0));
2306         matches.add(MatchEthernetType.IPV4);
2307         matches.add(new MatchIpv4Destination(subSplit[0], "32"));
2308
2309         List<ActionInfo> actionsInfos = new ArrayList<>();
2310
2311         // Set Eth Src and Eth Dst
2312         actionsInfos.add(new ActionMoveSourceDestinationEth());
2313         actionsInfos.add(new ActionSetFieldEthernetSource(routerMac));
2314
2315         // Move Ip Src to Ip Dst
2316         actionsInfos.add(new ActionMoveSourceDestinationIp());
2317         actionsInfos.add(new ActionSetSourceIp(subSplit[0], "32"));
2318
2319         // Set the ICMP type to 0 (echo reply)
2320         actionsInfos.add(new ActionSetIcmpType((short) 0));
2321
2322         actionsInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
2323
2324         actionsInfos.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2325
2326         List<InstructionInfo> instructions = new ArrayList<>();
2327
2328         instructions.add(new InstructionApplyActions(actionsInfos));
2329
2330         int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH;
2331         String flowRef = getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, vrfEntry.getLabel(), priority);
2332
2333         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef,
2334             0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions);
2335
2336         if (addOrRemove == NwConstants.ADD_FLOW) {
2337             mdsalManager.installFlow(flowEntity);
2338         } else {
2339             mdsalManager.removeFlow(flowEntity);
2340         }
2341     }
2342
2343     public void removeInterVPNLinkRouteFlows(final String interVpnLinkName,
2344                                              final boolean isVpnFirstEndPoint,
2345                                              final VrfEntry vrfEntry) {
2346         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
2347             && vrfEntry.getNextHopAddressList().size() == 1);
2348         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLinkName);
2349
2350         if (!interVpnLinkState.isPresent()) {
2351             LOG.warn("Could not find State for InterVpnLink {}", interVpnLinkName);
2352             return;
2353         }
2354
2355         List<BigInteger> targetDpns =
2356             isVpnFirstEndPoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
2357                 : interVpnLinkState.get().getSecondEndpointState().getDpId();
2358
2359         String nextHop = vrfEntry.getNextHopAddressList().get(0);
2360
2361         // delete from FIB
2362         //
2363         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2364         FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2365         Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef)).setTableId(NwConstants.L3_FIB_TABLE)
2366             .setFlowName(flowRef).build();
2367
2368         LOG.trace("Removing flow in FIB table for interVpnLink {} key {}",
2369             interVpnLinkName, flowRef);
2370
2371         for (BigInteger dpId : targetDpns) {
2372             LOG.debug("Removing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2373                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), nextHop,
2374                 dpId, interVpnLinkName);
2375
2376             mdsalManager.removeFlow(dpId, flow);
2377         }
2378
2379         // delete from LFIB
2380         //
2381         LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2382
2383         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
2384         for (BigInteger dpId : targetDpns) {
2385             LOG.debug("Removing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
2386                 vrfEntry.getDestPrefix(), vrfEntry.getLabel(), nextHop,
2387                 dpId, interVpnLinkName);
2388             makeLFibTableEntry(dpId, vrfEntry.getLabel(), null /* no instructions */,
2389                 LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, tx);
2390         }
2391         tx.submit();
2392     }
2393 }