vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.util.concurrent.FutureCallback;
  13 import com.google.common.util.concurrent.Futures;
  14 import com.google.common.util.concurrent.ListenableFuture;
  15
  16 import java.math.BigInteger;
  17 import java.net.Inet4Address;
  18 import java.net.InetAddress;
  19 import java.net.UnknownHostException;
  20 import java.util.ArrayList;
  21 import java.util.Arrays;
  22 import java.util.Collection;
  23 import java.util.Collections;
  24 import java.util.List;
  25 import java.util.concurrent.BlockingQueue;
  26 import java.util.concurrent.Callable;
  27 import java.util.concurrent.ExecutionException;
  28 import java.util.concurrent.Future;
  29 import java.util.concurrent.LinkedBlockingQueue;
  30 import java.util.function.Consumer;
  31 import java.util.stream.Collectors;
  32
  33 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  34 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  38 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  39 import org.opendaylight.genius.mdsalutil.ActionInfo;
  40 import org.opendaylight.genius.mdsalutil.FlowEntity;
  41 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  42 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  43 import org.opendaylight.genius.mdsalutil.MatchInfo;
  44 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  45 import org.opendaylight.genius.mdsalutil.NwConstants;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationEth;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationIp;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionSetIcmpType;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionSetSourceIp;
  59 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  60 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  61 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  62 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  63 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  64 import org.opendaylight.genius.mdsalutil.matches.MatchIcmpv4;
  65 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  67 import org.opendaylight.genius.mdsalutil.matches.MatchIpv6Destination;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.ServiceIndex;
  72 import org.opendaylight.genius.utils.batching.ActionableResource;
  73 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  74 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  75 import org.opendaylight.genius.utils.batching.ResourceHandler;
  76 import org.opendaylight.genius.utils.batching.SubTransaction;
  77 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
  78 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  79 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  80 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  81 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  82 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  83 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  84 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  85 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.Tunnel;
  86 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeInputBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeOutput;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.vrfentry.RoutePaths;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroutes;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.Vpn;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.VpnKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutes;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutesKey;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.RoutesKey;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 142 import org.opendaylight.yangtools.yang.binding.DataObject;
 143 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 144 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 145 import org.opendaylight.yangtools.yang.common.RpcResult;
 146 import org.slf4j.Logger;
 147 import org.slf4j.LoggerFactory;
 148
 149 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener>
 150     implements AutoCloseable, ResourceHandler {
 151
 152     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 153     private static final String FLOWID_PREFIX = "L3.";
 154     private static final int BATCH_SIZE = 1000;
 155     private static final int PERIODICITY = 500;
 156     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 157     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 158     private static final int LFIB_INTERVPN_PRIORITY = 15;
 159     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 160
 161     private final DataBroker dataBroker;
 162     private final IMdsalApiManager mdsalManager;
 163     private final NexthopManager nextHopManager;
 164     private final OdlInterfaceRpcService interfaceManager;
 165     private final IdManagerService idManager;
 166     private final IVpnLinkService ivpnLinkService;
 167
 168     List<SubTransaction> transactionObjects;
 169
 170     private static Integer batchSize;
 171     private static Integer batchInterval;
 172
 173     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 174     private final ResourceBatchingManager resourceBatchingManager;
 175
 176     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 177                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
 178                             final IdManagerService idManager, final IVpnLinkService ivpnLinkService) {
 179         super(VrfEntry.class, VrfEntryListener.class);
 180         this.dataBroker = dataBroker;
 181         this.mdsalManager = mdsalApiManager;
 182         this.nextHopManager = nexthopManager;
 183         this.interfaceManager = interfaceManager;
 184         this.idManager = idManager;
 185         this.ivpnLinkService = ivpnLinkService;
 186
 187         batchSize = Integer.getInteger("batch.size", BATCH_SIZE);
 188         batchInterval = Integer.getInteger("batch.wait.time", PERIODICITY);
 189         resourceBatchingManager = ResourceBatchingManager.getInstance();
 190         resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
 191         transactionObjects = new ArrayList<>();
 192     }
 193
 194     public void start() {
 195         LOG.info("{} start", getClass().getSimpleName());
 196         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 197     }
 198
 199     @Override
 200     protected VrfEntryListener getDataTreeChangeListener() {
 201         return VrfEntryListener.this;
 202     }
 203
 204     @Override
 205     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 206         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 207     }
 208
 209     @Override
 210     public DataBroker getResourceBroker() {
 211         return dataBroker;
 212     }
 213
 214     @Override
 215     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 216         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 217         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 218         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 219             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 220         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 221             createFibEntries(identifier, vrfEntry);
 222         } else {
 223             ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
 224             actResource.setAction(ActionableResource.CREATE);
 225             actResource.setInstanceIdentifier(identifier);
 226             actResource.setInstance(vrfEntry);
 227             vrfEntryBufferQ.add(actResource);
 228         }
 229         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 230         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 231             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 232     }
 233
 234     @Override
 235     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 236         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 237         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 238         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 239             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 240         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 241             deleteFibEntries(identifier, vrfEntry);
 242         } else {
 243             ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
 244             actResource.setAction(ActionableResource.DELETE);
 245             actResource.setInstanceIdentifier(identifier);
 246             actResource.setInstance(vrfEntry);
 247             vrfEntryBufferQ.add(actResource);
 248         }
 249         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 250         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 251             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 252     }
 253
 254     @Override
 255     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 256         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 257
 258         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 259         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
 260         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {}",
 261             rd, update.getDestPrefix(), update.getRoutePaths());
 262         // Handle BGP Routes first
 263         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 264             ActionableResource actResource = new ActionableResourceImpl(rd + update.getDestPrefix());
 265             actResource.setAction(ActionableResource.UPDATE);
 266             actResource.setInstanceIdentifier(identifier);
 267             actResource.setInstance(update);
 268             actResource.setOldInstance(original);
 269             vrfEntryBufferQ.add(actResource);
 270             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 271                 rd, update.getDestPrefix(), update.getRoutePaths());
 272             return;
 273         }
 274
 275         // Handle Vpn Interface driven Routes next (ie., STATIC and LOCAL)
 276         if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(update.getOrigin()))) {
 277             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 278             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 279             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 280
 281             // If original VRF Entry had nexthop null , but update VRF Entry
 282             // has nexthop , route needs to be created on remote Dpns
 283             if (((originalRoutePath == null) || (originalRoutePath.isEmpty())
 284                 && (updateRoutePath != null) && (!updateRoutePath.isEmpty()))) {
 285                 // TODO(vivek): Though ugly, Not handling this code now, as each
 286                 // tep add event will invoke flow addition
 287                 LOG.trace("Original VRF entry NH is null for destprefix {}. This event is IGNORED here.",
 288                     update.getDestPrefix());
 289                 return;
 290             }
 291
 292             // If original VRF Entry had valid nexthop , but update VRF Entry
 293             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 294             if (((updateRoutePath == null) || (updateRoutePath.isEmpty())
 295                 && (originalRoutePath != null) && (!originalRoutePath.isEmpty()))) {
 296                 LOG.trace("Original VRF entry had valid NH for destprefix {}. This event is IGNORED here.",
 297                     update.getDestPrefix());
 298                 return;
 299             }
 300             createFibEntries(identifier, update);
 301             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 302                 rd, update.getDestPrefix(), update.getRoutePaths());
 303             return;
 304         }
 305
 306         /* Handl all other route origins */
 307         createFibEntries(identifier, update);
 308
 309         LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 310             rd, update.getDestPrefix(), update.getRoutePaths());
 311     }
 312
 313     @Override
 314     public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 315                        Object original, Object update, List<SubTransaction> transactionObjects) {
 316         this.transactionObjects = transactionObjects;
 317         if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 318             createFibEntries(tx, identifier, (VrfEntry) update);
 319         }
 320     }
 321
 322     @Override
 323     public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 324                        Object vrfEntry, List<SubTransaction> transactionObjects) {
 325         this.transactionObjects = transactionObjects;
 326         if (vrfEntry instanceof VrfEntry) {
 327             createFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 328         }
 329     }
 330
 331     @Override
 332     public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 333                        Object vrfEntry, List<SubTransaction> transactionObjects) {
 334         this.transactionObjects = transactionObjects;
 335         if (vrfEntry instanceof VrfEntry) {
 336             deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 337         }
 338     }
 339
 340     @Override
 341     public int getBatchSize() {
 342         return batchSize;
 343     }
 344
 345     @Override
 346     public int getBatchInterval() {
 347         return batchInterval;
 348     }
 349
 350     @Override
 351     public LogicalDatastoreType getDatastoreType() {
 352         return LogicalDatastoreType.CONFIGURATION;
 353     }
 354
 355     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 356         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 357
 358         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 359         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 360         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 361             + " has null vpnId!");
 362
 363         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 364         final Long vpnId = vpnInstance.getVpnId();
 365         final String rd = vrfTableKey.getRouteDistinguisher();
 366         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 367         if (subnetRoute != null) {
 368             final long elanTag = subnetRoute.getElantag();
 369             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 370                 rd, vrfEntry.getDestPrefix(), elanTag);
 371             if (vpnToDpnList != null) {
 372                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 373                 dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(),
 374                     () -> {
 375                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 376                         for (final VpnToDpnList curDpn : vpnToDpnList) {
 377                             if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 378                                 installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 379                             }
 380                         }
 381                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 382                         futures.add(tx.submit());
 383                         return futures;
 384                     });
 385             }
 386             return;
 387         }
 388         // ping responder for router interfaces
 389         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnId, NwConstants.ADD_FLOW)) {
 390             return;
 391         }
 392
 393         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 394
 395         if (vpnToDpnList != null) {
 396             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 397             dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(),
 398                 () -> {
 399                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 400                     for (VpnToDpnList vpnDpn : vpnToDpnList) {
 401                         if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 402                             if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 403                                 createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 404                                     vrfTableKey, vrfEntry, tx);
 405                             }
 406                         }
 407                     }
 408                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 409                     futures.add(tx.submit());
 410                     return futures;
 411                 });
 412         }
 413
 414         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 415         if (optVpnUuid.isPresent()) {
 416             Optional<InterVpnLinkDataComposite> optInterVpnLink =
 417                 InterVpnLinkCache.getInterVpnLinkByVpnId(optVpnUuid.get());
 418             LOG.debug("InterVpnLink {} found in Cache: {}", optVpnUuid.get(), optInterVpnLink.isPresent());
 419             if (optInterVpnLink.isPresent()) {
 420                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
 421                 String vpnUuid = optVpnUuid.get();
 422                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 423                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 424                         // This is an static route that points to the other endpoint of an InterVpnLink
 425                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 426                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 427                         installInterVpnRouteInLFib(rd, vrfEntry);
 428                     }
 429                 });
 430             }
 431         }
 432     }
 433
 434
 435     /*
 436       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 437       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 438       provided by ResourceBatchingManager
 439      */
 440     private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
 441                                   final VrfEntry vrfEntry) {
 442         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 443
 444         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 445         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 446         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 447             + " has null vpnId!");
 448
 449         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 450         if (vpnToDpnList != null) {
 451             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 452                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 453                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 454                 }
 455             }
 456         }
 457     }
 458
 459     // FIXME: Refactoring needed here.
 460     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 461     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 462                                    int addOrRemove) {
 463         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 464         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 465
 466         String prefix = vrfEntry.getDestPrefix();
 467         List<String> nextHopsList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 468         // Label is used only for logging in subsequent method calls.
 469         //TODO : This label is not needed here. Can be removed. Hence using a default value.
 470         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(0L);
 471         String rd = vrfTableKey.getRouteDistinguisher();
 472         LOG.trace("leakRouteIfNeeded: srcVpnRd={}  prefix={}  nhList={}  label={}", rd, prefix, nextHopsList, label);
 473
 474         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 475         if (vpnInstance == null) {
 476             LOG.error("VPN Instance not available for route with prefix {} label {} nextHop {} RD {}. Returning...",
 477                       prefix, label, nextHopsList, rd);
 478             return;
 479         }
 480         String vpnUuid = vpnInstance.getVpnInstanceName();
 481         if (vpnUuid == null || vpnUuid.isEmpty()) {
 482             LOG.warn("Could not find suitable VPN UUID for rd={}.  vrfEntry=[prefix={}  nhList={}  label={}]",
 483                      rd, prefix, nextHopsList, label);
 484             return;
 485         }
 486
 487         ivpnLinkService.leakRouteIfNeeded(vpnUuid, prefix, nextHopsList, label.intValue(),
 488                                           RouteOrigin.value(vrfEntry.getOrigin()), addOrRemove);
 489     }
 490
 491     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 492         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 493         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 494         prefixBuilder.setDpnId(lri.getDpnId());
 495         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 496         prefixBuilder.setIpAddress(lri.getPrefix());
 497         // Increment the refCount here
 498         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 499             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 500         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 501         if (!isPresentInList) {
 502             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 503             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 504             vpnInstanceNames.add(vpnInstanceName);
 505             builder.setVpnInstanceList(vpnInstanceNames);
 506             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(),
 507                 FibUtil.DEFAULT_CALLBACK);
 508         } else {
 509             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 510         }
 511         return prefixBuilder.build();
 512     }
 513
 514     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 515                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 516         Boolean wrTxPresent = true;
 517         if (tx == null) {
 518             wrTxPresent = false;
 519             tx = dataBroker.newWriteOnlyTransaction();
 520         }
 521         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 522             List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 523             synchronized (label.toString().intern()) {
 524                 LabelRouteInfo lri = getLabelRouteInfo(label);
 525                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 526
 527                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 528                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 529                                 FibUtil.getVpnInstanceOpData(dataBroker, rd);
 530                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 531                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 532                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 533                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 534                             }
 535                         }
 536                     }
 537                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 538                             label, lri.getVpnInterfaceName(), lri.getDpnId());
 539                 }
 540             }
 541         });
 542         final List<InstructionInfo> instructions = new ArrayList<>();
 543         BigInteger subnetRouteMeta = ((BigInteger.valueOf(elanTag)).shiftLeft(32))
 544             .or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 545         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 546         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 547         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 548
 549         for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 550             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 551                 List<ActionInfo> actionsInfos = new ArrayList<>();
 552                 // reinitialize instructions list for LFIB Table
 553                 final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 554
 555                 actionsInfos.add(new ActionPopMpls());
 556                 LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 557                 LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 558                         MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 559                 LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 560
 561                 makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 562                         NwConstants.ADD_FLOW, tx);
 563             }
 564         }
 565         if (!wrTxPresent) {
 566             tx.submit();
 567         }
 568     }
 569
 570     /*
 571      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 572      * LportDispatcher table (via table 80)
 573      */
 574     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 575         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 576         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 577         // packet is commuted from Vpn2 to Vpn1.
 578         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
 579         if (!vpnNameOpc.isPresent()) {
 580             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 581             return;
 582         }
 583
 584         String vpnName = vpnNameOpc.get();
 585         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
 586         boolean interVpnLinkFound = false;
 587         for (InterVpnLink interVpnLink : interVpnLinks) {
 588             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 589             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 590                 && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 591             if (vpnIs1stEndpoint || vpnIs2ndEndpoint) {
 592                 interVpnLinkFound = true;
 593
 594                 Optional<InterVpnLinkState> vpnLinkState =
 595                     FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 596                 if (!vpnLinkState.isPresent()
 597                     || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active)) {
 598                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 599                         interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 600                         interVpnLink.getSecondEndpoint().getVpnUuid().getValue());
 601                     return;
 602                 }
 603
 604                 List<BigInteger> targetDpns = vpnIs1stEndpoint ? vpnLinkState.get().getFirstEndpointState().getDpId()
 605                     : vpnLinkState.get().getSecondEndpointState().getDpId();
 606                 Long lportTag = vpnIs1stEndpoint ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 607                     : vpnLinkState.get().getFirstEndpointState().getLportTag();
 608
 609                 LOG.trace("Installing flow in LFIB table for interVpnLink {}", interVpnLink.getName());
 610
 611                 for (BigInteger dpId : targetDpns) {
 612                     List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls());
 613
 614                     List<InstructionInfo> instructions =
 615                         Arrays.asList(new InstructionApplyActions(actionsInfos),
 616                             new InstructionWriteMetadata(
 617                                 MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 618                                     ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 619                                         NwConstants.L3VPN_SERVICE_INDEX)),
 620                                 MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 621                             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 622
 623                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(interVpnRoutePathLabel -> {
 624                         List<String> interVpnNextHopList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 625                         LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for "
 626                                 + "InterVpnLink {} in LFIB",
 627                                 vrfEntry.getDestPrefix(), interVpnRoutePathLabel, interVpnNextHopList,
 628                                 dpId, interVpnLink.getName());
 629
 630                         makeLFibTableEntry(dpId, interVpnRoutePathLabel, instructions, LFIB_INTERVPN_PRIORITY,
 631                                 NwConstants.ADD_FLOW, null);
 632                     });
 633                 }
 634
 635                 break;
 636             }
 637         }
 638
 639         if (!interVpnLinkFound) {
 640             LOG.warn("VrfEntry=[prefix={} route-paths={}] for VPN {} has origin INTERVPN but "
 641                 + "no InterVpnLink could be found",
 642                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), rd);
 643         }
 644     }
 645
 646
 647     /*
 648      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 649      */
 650     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 651                                                final VrfEntry vrfEntry, long vpnTag) {
 652         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 653         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 654             && vrfEntry.getRoutePaths().size() == 1);
 655         String destination = vrfEntry.getDestPrefix();
 656         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 657         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 658
 659         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 660         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 661         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 662         if (interVpnLink.getState().or(State.Error) != State.Active) {
 663             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 664                 destination, nextHop, interVpnLinkName);
 665             return;
 666         }
 667
 668         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 669         if (!optOtherEndpointLportTag.isPresent()) {
 670             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 671                 vpnUuid, interVpnLinkName);
 672             return;
 673         }
 674
 675         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 676         if (targetDpns.isEmpty()) {
 677             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 678                 vpnUuid, interVpnLinkName);
 679             return;
 680         }
 681
 682         String[] values = destination.split("/");
 683         String destPrefixIpAddress = values[0];
 684         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 685
 686         List<MatchInfo> matches = new ArrayList<>();
 687         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 688         matches.add(MatchEthernetType.IPV4);
 689
 690         if (prefixLength != 0) {
 691             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 692         }
 693
 694         List<Instruction> instructions =
 695             Arrays.asList(new InstructionWriteMetadata(
 696                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 697                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 698                             .L3VPN_SERVICE_INDEX)),
 699                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 700                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 701
 702         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 703         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 704         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 705             COOKIE_VM_FIB_TABLE, matches, instructions);
 706
 707         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 708             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 709
 710         for (BigInteger dpId : targetDpns) {
 711
 712             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 713                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 714                 dpId, interVpnLink.getInterVpnLinkName());
 715
 716             mdsalManager.installFlow(dpId, flowEntity);
 717         }
 718     }
 719
 720
 721     // TODO Clean up the exception handling
 722     @SuppressWarnings("checkstyle:IllegalCatch")
 723     private <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 724                                                     InstanceIdentifier<T> path) {
 725
 726         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 727
 728         Optional<T> result = Optional.absent();
 729         try {
 730             result = tx.read(datastoreType, path).get();
 731         } catch (Exception e) {
 732             throw new RuntimeException(e);
 733         }
 734
 735         return result;
 736     }
 737
 738     private List<BigInteger> getDpnIdForPrefix(DataBroker broker, Long vpnId, String rd, VrfEntry vrfEntry) {
 739         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 740         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, vrfEntry.getDestPrefix());
 741
 742         if (localNextHopInfo == null) {
 743             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 744             Routes extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 745             if (extraRoute != null) {
 746                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 747                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 748                     if (nextHopIp != null) {
 749                         localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, nextHopIp + "/32");
 750                         if (localNextHopInfo != null) {
 751                             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 752                         }
 753                     }
 754                 }
 755             }
 756         } else {
 757             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 758         }
 759
 760         return returnLocalDpnId;
 761     }
 762
 763     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 764         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 765         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
 766         String localNextHopIP = vrfEntry.getDestPrefix();
 767
 768         if (localNextHopInfo == null) {
 769             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 770             Routes extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 771             if (extraRoute != null) {
 772                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 773                     LOG.info("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 774                     if (nextHopIp != null) {
 775                         localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32");
 776                         localNextHopIP = nextHopIp + "/32";
 777                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 778                             rd, vrfEntry, vpnId);
 779                         returnLocalDpnId.add(dpnId);
 780                     }
 781                 }
 782             }
 783             if (localNextHopInfo == null) {
 784             /* imported routes case */
 785                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 786                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 787                     if (optionalLabel.isPresent()) {
 788                         Long label = optionalLabel.get();
 789                         List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 790                         synchronized (label.toString().intern()) {
 791                             LabelRouteInfo lri = getLabelRouteInfo(label);
 792                             if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 793                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 794                                         FibUtil.getVpnInstanceOpData(dataBroker, rd);
 795                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
 796                                     String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 797                                     if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 798                                         localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 799                                         localNextHopIP = lri.getPrefix();
 800                                     } else {
 801                                         localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 802                                         localNextHopIP = lri.getPrefix();
 803                                     }
 804                                 }
 805                                 if (localNextHopInfo != null) {
 806                                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 807                                             label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 808                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 809                                             vpnId, rd, vrfEntry, lri.getParentVpnid());
 810                                     returnLocalDpnId.add(dpnId);
 811                                 }
 812                             }
 813                         }
 814                     }
 815                 }
 816             }
 817         } else {
 818             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 819             returnLocalDpnId.add(dpnId);
 820         }
 821
 822         return returnLocalDpnId;
 823     }
 824
 825     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 826                                                 final Long vpnId, final String rd,
 827                                                 final VrfEntry vrfEntry, Long parentVpnId) {
 828         if (localNextHopInfo != null) {
 829             final BigInteger dpnId = localNextHopInfo.getDpnId();
 830             if (!isVpnPresentInDpn(rd, dpnId)) {
 831                 LOG.error("The vpnName with vpnId {} rd {} is not available on dpn {}", vpnId, rd, dpnId.toString());
 832                 return BigInteger.ZERO;
 833             }
 834
 835             if (Boolean.TRUE.equals(localNextHopInfo.isNatPrefix())) {
 836                 LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip local dpn {} FIB processing",
 837                         vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 838                 return dpnId;
 839             }
 840
 841             String jobKey = "FIB-" + vpnId.toString() + "-" + dpnId.toString() + "-" + vrfEntry.getDestPrefix();
 842             final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId,
 843                     localNextHopInfo.getVpnInterfaceName(), localNextHopIP, vrfEntry.getDestPrefix(),
 844                     vrfEntry.getGatewayMacAddress(), jobKey);
 845             if (groupId == 0) {
 846                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 847                     vrfEntry.getDestPrefix(), rd, localNextHopInfo.getVpnInterfaceName(), dpnId.toString());
 848                 return BigInteger.ZERO;
 849             }
 850             final List<InstructionInfo> instructions = Collections.singletonList(
 851                 new InstructionApplyActions(
 852                     Collections.singletonList(new ActionGroup(groupId))));
 853             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 854                 new InstructionApplyActions(
 855                     Arrays.asList(new ActionPopMpls(), new ActionGroup(groupId))));
 856             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 857             List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 858             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 859                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 860                     dpnId, localNextHopInfo.getVpnInterfaceName(), optLabel);
 861             } else {
 862                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. "
 863                     + "LFib and Terminating table entries will not be created.",
 864                     rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 865             }
 866             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 867
 868             dataStoreCoordinator.enqueueJob(jobKey,
 869                 () -> {
 870                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 871                     makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 872                     optLabel.ifPresent(label -> {
 873                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 874                             makeLFibTableEntry(dpnId, label, lfibinstructions,
 875                                     DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 876                             makeTunnelTableEntry(dpnId, label, groupId, tx);
 877                         }
 878                     });
 879                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 880                     futures.add(tx.submit());
 881                     return futures;
 882                 });
 883             return dpnId;
 884         }
 885         return BigInteger.ZERO;
 886     }
 887
 888     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId) {
 889         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 890         return FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id).isPresent();
 891     }
 892
 893     private LabelRouteInfo getLabelRouteInfo(Long label) {
 894         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 895             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 896         Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 897         if (opResult.isPresent()) {
 898             return opResult.get();
 899         }
 900         return null;
 901     }
 902
 903     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
 904         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 905         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 906             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 907         if (lri == null) {
 908             return true;
 909         }
 910         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 911             ? lri.getVpnInstanceList() : new ArrayList<>();
 912         if (vpnInstancesList.contains(vpnInstanceName)) {
 913             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 914             vpnInstancesList.remove(vpnInstanceName);
 915         }
 916         if (vpnInstancesList.size() == 0) {
 917             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 918             if (tx != null) {
 919                 tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
 920             } else {
 921                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 922             }
 923             return true;
 924         } else {
 925             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 926             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 927             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(),
 928                 FibUtil.DEFAULT_CALLBACK);
 929         }
 930         return false;
 931     }
 932
 933     private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 934                                       WriteTransaction tx) {
 935         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 936
 937         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 938
 939         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 940             dpId, label, groupId);
 941     }
 942
 943     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 944                                                 WriteTransaction tx) {
 945         List<MatchInfo> mkMatches = new ArrayList<>();
 946
 947         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 948             destDpId, label, actionsInfos);
 949
 950         // Matching metadata
 951         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 952         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 953
 954         List<InstructionInfo> mkInstructions = new ArrayList<>();
 955         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 956
 957         FlowEntity terminatingServiceTableFlowEntity =
 958             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 959             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
 960                 String.format("%s:%d", "TST Flow Entry ", label),
 961             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
 962
 963         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
 964
 965         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 966
 967         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 968         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 969             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 970             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
 971             .child(Flow.class, flowKey).build();
 972         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(), true);
 973     }
 974
 975     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 976         FlowEntity flowEntity;
 977         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
 978         List<MatchInfo> mkMatches = new ArrayList<>();
 979         // Matching metadata
 980         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 981         flowEntity = MDSALUtil.buildFlowEntity(dpId,
 982             NwConstants.INTERNAL_TUNNEL_TABLE,
 983             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
 984             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
 985             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
 986         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
 987         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
 988         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 989             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 990             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 991
 992         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 993         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
 994     }
 995
 996     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 997         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 998         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
 999         String localNextHopIP = vrfEntry.getDestPrefix();
1000
1001         if (localNextHopInfo == null) {
1002             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1003             Routes extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1004             if (extraRoute != null) {
1005                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1006                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1007                     if (nextHopIp != null) {
1008                         localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
1009                         localNextHopIP = nextHopIp + "/32";
1010                         BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1011                             vpnId, rd, vrfEntry, true /*isExtraRoute*/);
1012                         if (!dpnId.equals(BigInteger.ZERO)) {
1013                             returnLocalDpnId.add(dpnId);
1014                         }
1015                     }
1016                 }
1017             }
1018
1019             if (localNextHopInfo == null) {
1020                 /* Imported VRF entry */
1021                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1022                 if (optionalLabel.isPresent()) {
1023                     Long label = optionalLabel.get();
1024                     List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1025                     LabelRouteInfo lri = getLabelRouteInfo(label);
1026                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1027                         VpnNexthopBuilder vpnNexthopBuilder = new VpnNexthopBuilder();
1028                         vpnNexthopBuilder.setDpnId(lri.getDpnId());
1029                         BigInteger dpnId = checkDeleteLocalFibEntry(vpnNexthopBuilder.build(), localNextHopIP,
1030                                 vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1031                         if (!dpnId.equals(BigInteger.ZERO)) {
1032                             returnLocalDpnId.add(dpnId);
1033                         }
1034                     }
1035                 }
1036             }
1037
1038         } else {
1039             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1040                 vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1041             if (!dpnId.equals(BigInteger.ZERO)) {
1042                 returnLocalDpnId.add(dpnId);
1043             }
1044         }
1045
1046         return returnLocalDpnId;
1047     }
1048
1049     private BigInteger checkDeleteLocalFibEntry(VpnNexthop localNextHopInfo, final String localNextHopIP,
1050                                                 final Long vpnId, final String rd,
1051                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
1052         if (localNextHopInfo != null) {
1053             final BigInteger dpnId = localNextHopInfo.getDpnId();
1054             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1055             dataStoreCoordinator.enqueueJob("FIB-" + vpnId.toString() + "-"
1056                     + dpnId.toString() + "-" + vrfEntry.getDestPrefix(),
1057                 () -> {
1058                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1059                     makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
1060                         NwConstants.DEL_FLOW, tx);
1061                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1062                         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1063                             makeLFibTableEntry(dpnId, label, null /* instructions */,
1064                                     DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1065                             removeTunnelTableEntry(dpnId, label, tx);
1066                         });
1067                     }
1068                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1069                     futures.add(tx.submit());
1070                     return futures;
1071                 });
1072             //TODO: verify below adjacency call need to be optimized (?)
1073             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1074             return dpnId;
1075         }
1076         return BigInteger.ZERO;
1077     }
1078
1079     static  InstanceIdentifier<Routes> getVpnToExtrarouteIdentifier(String vpnName, String vrfId, String ipPrefix) {
1080         return InstanceIdentifier.builder(VpnToExtraroutes.class)
1081                 .child(Vpn.class, new VpnKey(vpnName)).child(ExtraRoutes.class,
1082                         new ExtraRoutesKey(vrfId)).child(Routes.class, new RoutesKey(ipPrefix)).build();
1083     }
1084
1085     public Routes getVpnToExtraroute(String vpnRd, String destPrefix) {
1086         Optional<String> optVpnName = FibUtil.getVpnNameFromRd(dataBroker, vpnRd);
1087         if (optVpnName.isPresent()) {
1088             InstanceIdentifier<Routes> vpnExtraRoutesId = getVpnToExtrarouteIdentifier(
1089                     optVpnName.get(), vpnRd, destPrefix);
1090             return FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, vpnExtraRoutesId).orNull();
1091         }
1092         return null;
1093     }
1094
1095     private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
1096         try {
1097             Future<RpcResult<GetTunnelTypeOutput>> result = interfaceManager.getTunnelType(
1098                 new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
1099             RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
1100             if (!rpcResult.isSuccessful()) {
1101                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
1102             } else {
1103                 return rpcResult.getResult().getTunnelType();
1104             }
1105
1106         } catch (InterruptedException | ExecutionException e) {
1107             LOG.warn("Exception when getting tunnel interface Id for tunnel type", e);
1108         }
1109
1110         return null;
1111
1112     }
1113
1114     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1115                                       final VrfEntry vrfEntry, WriteTransaction tx) {
1116         Boolean wrTxPresent = true;
1117         if (tx == null) {
1118             wrTxPresent = false;
1119             tx = dataBroker.newWriteOnlyTransaction();
1120         }
1121         String rd = vrfTableKey.getRouteDistinguisher();
1122         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1123             vrfEntry.getDestPrefix(), rd, remoteDpnId);
1124
1125         List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1126         if (adjacencyResults.isEmpty()) {
1127             LOG.error("Could not get interface for route-paths: {} in vpn {}",
1128                 vrfEntry.getRoutePaths(), rd);
1129             LOG.warn("Failed to add Route: {} in vpn: {}",
1130                 vrfEntry.getDestPrefix(), rd);
1131             return;
1132         }
1133
1134         for (AdjacencyResult adjacencyResult : adjacencyResults) {
1135             List<ActionInfo> actionInfos = new ArrayList<>();
1136             String egressInterface = adjacencyResult.getInterfaceName();
1137             if (Tunnel.class.equals(adjacencyResult.getInterfaceType())) {
1138                 addTunnelInterfaceActions(egressInterface, vpnId, vrfEntry, actionInfos);
1139             } else {
1140                 addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1141             }
1142             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(egressInterface,
1143                     actionInfos.size());
1144             if (egressActions.isEmpty()) {
1145                 LOG.error("Failed to retrieve egress action for prefix {} route-paths {} interface {}. "
1146                     + "Aborting remote FIB entry creation.",
1147                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
1148                 return;
1149             }
1150             actionInfos.addAll(egressActions);
1151             List<InstructionInfo> instructions = new ArrayList<>();
1152             instructions.add(new InstructionApplyActions(actionInfos));
1153             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1154         }
1155         if (!wrTxPresent) {
1156             tx.submit();
1157         }
1158         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1159     }
1160
1161     private void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, List<ActionInfo> actionInfos) {
1162         String ipPrefix = vrfEntry.getDestPrefix();
1163         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, ipPrefix);
1164         if (prefixInfo == null) {
1165             LOG.debug("No prefix info found for prefix {}", ipPrefix);
1166             return;
1167         }
1168
1169         String ifName = prefixInfo.getVpnInterfaceName();
1170         if (ifName == null) {
1171             LOG.debug("Failed to get VPN interface for prefix {}", ipPrefix);
1172             return;
1173         }
1174
1175         String macAddress = FibUtil.getMacAddressFromPrefix(dataBroker, ifName, ipPrefix);
1176         if (macAddress == null) {
1177             LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix);
1178             return;
1179         }
1180
1181         actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(), new MacAddress(macAddress)));
1182     }
1183
1184     private void addTunnelInterfaceActions(String tunnelInterface, long vpnId, VrfEntry vrfEntry,
1185                                            List<ActionInfo> actionInfos) {
1186         Class<? extends TunnelTypeBase> tunnelType = getTunnelType(tunnelInterface);
1187         java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1188         if (!optionalLabel.isPresent()) {
1189             LOG.warn("RoutePaths not available");
1190             return;
1191         }
1192         long label = optionalLabel.get();
1193         if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
1194             LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1195             actionInfos.add(new ActionPushMpls());
1196             actionInfos.add(new ActionSetFieldMplsLabel(label));
1197             actionInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
1198         } else {
1199             BigInteger tunnelId;
1200             // FIXME vxlan vni bit set is not working properly with OVS.need to
1201             // revisit
1202             if (tunnelType.equals(TunnelTypeVxlan.class)) {
1203                 tunnelId = BigInteger.valueOf(label);
1204             } else {
1205                 tunnelId = BigInteger.valueOf(label);
1206             }
1207
1208             LOG.debug("adding set tunnel id action for label {}", label);
1209             actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
1210             addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1211         }
1212     }
1213
1214     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1215         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1216         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1217         if (dpnInVpn.isPresent()) {
1218             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1219             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1220
1221             if (vpnInterfaces.remove(currVpnInterface)) {
1222                 if (vpnInterfaces.isEmpty()) {
1223                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1224                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1225                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
1226                 } else {
1227                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1228                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1229                         VpnInterfaces.class,
1230                         new VpnInterfacesKey(intfName)));
1231                 }
1232             }
1233         }
1234     }
1235
1236     private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
1237     /* Get interface info from prefix to interface mapping;
1238         Use the interface info to get the corresponding vpn interface op DS entry,
1239         remove the adjacency corresponding to this fib entry.
1240         If adjacency removed is the last adjacency, clean up the following:
1241          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1242          - prefix to interface entry
1243          - vpn interface op DS
1244      */
1245         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1246         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1247         Routes extraRoute = null;
1248         if (prefixInfo == null) {
1249             extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1250             if (extraRoute != null) {
1251                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1252                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1253
1254                     if (nextHopIp != null) {
1255                         prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32");
1256                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1257                     }
1258                 }
1259             }
1260             if (prefixInfo == null) {
1261                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1262                 if (optionalLabel.isPresent()) {
1263                     Long label = optionalLabel.get();
1264                     List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1265                     LabelRouteInfo lri = getLabelRouteInfo(label);
1266                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1267                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1268                         prefixBuilder.setDpnId(lri.getDpnId());
1269                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1270                         prefixBuilder.setIpAddress(lri.getPrefix());
1271                         prefixInfo = prefixBuilder.build();
1272                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1273                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1274                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1275                     }
1276                 }
1277             }
1278         } else {
1279             checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1280         }
1281     }
1282
1283     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1284                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1285
1286         if (prefixInfo == null) {
1287             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1288             return; //Don't have any info for this prefix (shouldn't happen); need to return
1289         }
1290
1291         if (Boolean.TRUE.equals(prefixInfo.isNatPrefix())) {
1292             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1293                     vrfEntry.getDestPrefix(), vpnId, rd);
1294             return;
1295         }
1296
1297         String ifName = prefixInfo.getVpnInterfaceName();
1298         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1299         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1300             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1301     }
1302
1303     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1304         Prefixes prefixInfo;
1305         Long vpnId;
1306         String rd;
1307         VrfEntry vrfEntry;
1308         Routes extraRoute;
1309
1310         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1311                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1312             this.prefixInfo = prefixInfo;
1313             this.vpnId = vpnId;
1314             this.rd = rd;
1315             this.vrfEntry = vrfEntry;
1316             this.extraRoute = extraRoute;
1317         }
1318
1319         @Override
1320         public List<ListenableFuture<Void>> call() throws Exception {
1321             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1322             // to call the respective helpers.
1323             WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1324
1325             //First Cleanup LabelRouteInfo
1326             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1327                 List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1328                 synchronized (label.toString().intern()) {
1329                     LabelRouteInfo lri = getLabelRouteInfo(label);
1330                     if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1331                             && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1332                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1333                                 FibUtil.getVpnInstanceOpData(dataBroker, rd);
1334                         String vpnInstanceName = "";
1335                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1336                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1337                         }
1338                         boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, writeOperTxn);
1339                         if (lriRemoved) {
1340                             String parentRd = lri.getParentVpnRd();
1341                             FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1342                                     FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1343                         }
1344                     } else {
1345                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1346                                 FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1347                     }
1348                 }
1349             });
1350             String ifName = prefixInfo.getVpnInterfaceName();
1351             Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1352                 FibUtil.getVpnInterfaceIdentifier(ifName));
1353             if (optvpnInterface.isPresent()) {
1354                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1355                 if (vpnId != associatedVpnId) {
1356                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1357                         vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1358                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1359                     return null;
1360                 } else {
1361                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1362                         vrfEntry.getDestPrefix(), associatedVpnId);
1363                 }
1364             }
1365             if (extraRoute != null) {
1366                 Optional<String> optVpnName = FibUtil.getVpnNameFromRd(dataBroker, rd);
1367                 if (optVpnName.isPresent()) {
1368                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL,
1369                             getVpnToExtrarouteIdentifier(optVpnName.get(), rd, vrfEntry.getDestPrefix()));
1370                 }
1371             }
1372             Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1373                 FibUtil.getAdjListPath(ifName));
1374             int numAdj = 0;
1375             if (optAdjacencies.isPresent()) {
1376                 numAdj = optAdjacencies.get().getAdjacency().size();
1377             }
1378             //remove adjacency corr to prefix
1379             if (numAdj > 1) {
1380                 boolean isPrimaryAdj = false;
1381                 List<Adjacency> adjacencyList = optAdjacencies.get().getAdjacency();
1382                 for (Adjacency adj : adjacencyList) {
1383                     if (adj.getIpAddress().equals(vrfEntry.getDestPrefix())) {
1384                         isPrimaryAdj = adj.isPrimaryAdjacency();
1385                         break;
1386                     }
1387                 }
1388                 //We should only delete secondary adjacencies, as primary adjacency is used to
1389                 //cleanup prefix-to-interface in the subscriber vpnInterfaceOpListener
1390                 if (!isPrimaryAdj) {
1391                     LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId,
1392                             vrfEntry.getDestPrefix());
1393                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL,
1394                             FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1395                 }
1396             }
1397             if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
1398                 //clean up the vpn interface from DpnToVpn list
1399                 LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1400                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, FibUtil.getVpnInterfaceIdentifier(ifName));
1401             }
1402             List<ListenableFuture<Void>> futures = new ArrayList<>();
1403             futures.add(writeOperTxn.submit());
1404             return futures;
1405         }
1406     }
1407
1408     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1409         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1410         final String rd = vrfTableKey.getRouteDistinguisher();
1411         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1412         if (vpnInstance == null) {
1413             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1414             return;
1415         }
1416         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1417         long elanTag = 0L;
1418         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1419         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1420         List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1421         if (subnetRoute != null) {
1422             elanTag = subnetRoute.getElantag();
1423             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1424                 rd, vrfEntry.getDestPrefix(), elanTag);
1425             if (vpnToDpnList != null) {
1426                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1427                 dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(),
1428                     () -> {
1429                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1430
1431                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1432
1433                             makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1434                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1435                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1436                                 optionalLabel.ifPresent(label -> {
1437                                     makeLFibTableEntry(curDpn.getDpnId(), label, null,
1438                                             DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1439                                 });
1440                             }
1441                         }
1442                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1443                         futures.add(tx.submit());
1444                         return futures;
1445                     });
1446             }
1447             optionalLabel.ifPresent(label -> {
1448                 synchronized (label.toString().intern()) {
1449                     LabelRouteInfo lri = getLabelRouteInfo(label);
1450                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1451                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1452                                 FibUtil.getVpnInstanceOpData(dataBroker, rd);
1453                         String vpnInstanceName = "";
1454                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1455                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1456                         }
1457                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1458                         if (lriRemoved) {
1459                             String parentRd = lri.getParentVpnRd();
1460                             FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1461                                     FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1462                             LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as "
1463                                     + "labelRouteInfo cleared", label, rd,
1464                                     vrfEntry.getDestPrefix());
1465                         }
1466                     } else {
1467                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1468                                 FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1469                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1470                                 label, rd, vrfEntry.getDestPrefix());
1471                     }
1472                 }
1473             });
1474             return;
1475         }
1476         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnInstance.getVpnId(), NwConstants.DEL_FLOW)) {
1477             return;
1478         }
1479
1480         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1481             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1482         if (vpnToDpnList != null) {
1483             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1484             dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(),
1485                 () -> {
1486                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1487
1488                     if (localDpnIdList.size() <= 0) {
1489                         for (VpnToDpnList curDpn : vpnToDpnList) {
1490                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1491                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1492                                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1493                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1494                                 }
1495                             } else {
1496                                 deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1497                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1498                             }
1499                         }
1500                     } else {
1501                         for (BigInteger localDpnId : localDpnIdList) {
1502                             for (VpnToDpnList curDpn : vpnToDpnList) {
1503                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1504                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1505                                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1506                                             deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1507                                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1508                                         }
1509                                     } else {
1510                                         deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1511                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1512                                     }
1513                                 }
1514                             }
1515                         }
1516                     }
1517                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1518                     futures.add(tx.submit());
1519                     return futures;
1520                 });
1521         }
1522
1523         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1524         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1525         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1526
1527         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1528         // of the interVpnLink.
1529         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(this.dataBroker, rd);
1530         if (optVpnUuid.isPresent()) {
1531             String vpnUuid = optVpnUuid.get();
1532             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1533                 Optional<InterVpnLinkDataComposite> optInterVpnLink =
1534                         InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1535                 if (optInterVpnLink.isPresent()) {
1536                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1537                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1538                         // This is route that points to the other endpoint of an InterVpnLink
1539                         // In that case, we should look for the FIB table pointing to
1540                         // LPortDispatcher table and remove it.
1541                         removeInterVPNLinkRouteFlows(interVpnLink.getInterVpnLinkName(),
1542                                 interVpnLink.isFirstEndpointVpnName(rd),
1543                                 vrfEntry);
1544                     }
1545                 }
1546             });
1547         }
1548
1549     }
1550
1551     /*
1552       Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1553       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1554       provided by ResourceBatchingManager
1555      */
1556     private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
1557                                   final VrfEntry vrfEntry) {
1558         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1559
1560         final String rd = vrfTableKey.getRouteDistinguisher();
1561         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1562         if (vpnInstance == null) {
1563             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1564             return;
1565         }
1566         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1567         if (vpnToDpnList != null) {
1568             for (VpnToDpnList curDpn : vpnToDpnList) {
1569                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1570                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(),
1571                         vrfTableKey, vrfEntry, writeTx);
1572                 }
1573             }
1574         }
1575     }
1576
1577     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1578                                   final long vpnId, final VrfTablesKey vrfTableKey,
1579                                   final VrfEntry vrfEntry, WriteTransaction tx) {
1580
1581         Boolean wrTxPresent = true;
1582         if (tx == null) {
1583             wrTxPresent = false;
1584             tx = dataBroker.newWriteOnlyTransaction();
1585         }
1586
1587         LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
1588             vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
1589         String rd = vrfTableKey.getRouteDistinguisher();
1590
1591         if (localDpnId != null) {
1592             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1593             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1594             return;
1595         }
1596
1597         // below two reads are kept as is, until best way is found to identify dpnID
1598         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1599         Routes extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1600
1601         if (localNextHopInfo == null && extraRoute != null) {
1602             // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1603             for (String nextHopIp : extraRoute.getNexthopIpList()) {
1604                 localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
1605                 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1606             }
1607         } else {
1608             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1609         }
1610         if (!wrTxPresent) {
1611             tx.submit();
1612         }
1613     }
1614
1615     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1616                                            VrfEntry vrfEntry, String rd, WriteTransaction tx) {
1617         boolean isRemoteRoute = true;
1618         if (localNextHopInfo != null) {
1619             isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId());
1620         }
1621         if (isRemoteRoute) {
1622             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1623             return true;
1624         } else {
1625             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}",
1626                 rd, vrfEntry.getDestPrefix(), remoteDpnId);
1627             return false;
1628         }
1629     }
1630
1631     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry, String rd, WriteTransaction tx) {
1632         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1633         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1634     }
1635
1636     private long get(byte[] rawIpAddress) {
1637         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1638             + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1639     }
1640
1641     private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1642                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1643         Boolean wrTxPresent = true;
1644         if (tx == null) {
1645             wrTxPresent = false;
1646             tx = dataBroker.newWriteOnlyTransaction();
1647         }
1648
1649         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1650         String[] values = vrfEntry.getDestPrefix().split("/");
1651         String ipAddress = values[0];
1652         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1653         if (addOrRemove == NwConstants.ADD_FLOW) {
1654             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1655         } else {
1656             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1657         }
1658         InetAddress destPrefix;
1659         try {
1660             destPrefix = InetAddress.getByName(ipAddress);
1661         } catch (UnknownHostException e) {
1662             LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1663             return;
1664         }
1665
1666         List<MatchInfo> matches = new ArrayList<>();
1667
1668         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
1669
1670         if (destPrefix instanceof Inet4Address) {
1671             matches.add(MatchEthernetType.IPV4);
1672             if (prefixLength != 0) {
1673                 matches.add(new MatchIpv4Destination(destPrefix.getHostAddress(), Integer.toString(prefixLength)));
1674             }
1675         } else {
1676             matches.add(MatchEthernetType.IPV6);
1677             if (prefixLength != 0) {
1678                 matches.add(new MatchIpv6Destination(destPrefix.getHostAddress() + "/" + prefixLength));
1679             }
1680         }
1681
1682         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1683         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1684         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority,
1685             flowRef, 0, 0,
1686             COOKIE_VM_FIB_TABLE, matches, instructions);
1687
1688         Flow flow = flowEntity.getFlowBuilder().build();
1689         String flowId = flowEntity.getFlowId();
1690         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1691         Node nodeDpn = buildDpnNode(dpId);
1692
1693         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1694             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1695             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1696
1697         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1698             SubTransaction subTransaction = new SubTransactionImpl();
1699             if (addOrRemove == NwConstants.ADD_FLOW) {
1700                 subTransaction.setInstanceIdentifier(flowInstanceId);
1701                 subTransaction.setInstance(flow);
1702                 subTransaction.setAction(SubTransaction.CREATE);
1703             } else {
1704                 subTransaction.setInstanceIdentifier(flowInstanceId);
1705                 subTransaction.setAction(SubTransaction.DELETE);
1706             }
1707             transactionObjects.add(subTransaction);
1708         }
1709
1710         if (addOrRemove == NwConstants.ADD_FLOW) {
1711             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
1712         } else {
1713             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1714         }
1715
1716         if (!wrTxPresent) {
1717             tx.submit();
1718         }
1719     }
1720
1721     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1722     private Node buildDpnNode(BigInteger dpnId) {
1723         NodeId nodeId = new NodeId("openflow:" + dpnId);
1724         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1725
1726         return nodeDpn;
1727     }
1728
1729     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1730                                     int addOrRemove, WriteTransaction tx) {
1731         Boolean wrTxPresent = true;
1732         if (tx == null) {
1733             wrTxPresent = false;
1734             tx = dataBroker.newWriteOnlyTransaction();
1735         }
1736
1737         List<MatchInfo> matches = new ArrayList<>();
1738         matches.add(MatchEthernetType.MPLS_UNICAST);
1739         matches.add(new MatchMplsLabel(label));
1740
1741         // Install the flow entry in L3_LFIB_TABLE
1742         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1743
1744         FlowEntity flowEntity;
1745         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1746             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1747         Flow flow = flowEntity.getFlowBuilder().build();
1748         String flowId = flowEntity.getFlowId();
1749         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1750         Node nodeDpn = buildDpnNode(dpId);
1751         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1752             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1753             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1754
1755         if (addOrRemove == NwConstants.ADD_FLOW) {
1756             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
1757         } else {
1758             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1759         }
1760         if (!wrTxPresent) {
1761             tx.submit();
1762         }
1763
1764         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1765             dpId, label, instructions, flowKey, (NwConstants.ADD_FLOW == addOrRemove) ? "ADDED" : "REMOVED");
1766     }
1767
1768     private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress,
1769                                       final String ipPrefixAddress) {
1770         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}", dpId, vpnId, ipAddress);
1771         try {
1772             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress);
1773         } catch (NullPointerException e) {
1774             LOG.trace("", e);
1775         }
1776     }
1777
1778     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1779                                     final FutureCallback<List<Void>> callback) {
1780         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1781         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1782         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1783         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1784         if (!vrfTable.isPresent()) {
1785             LOG.warn("VRF Table not yet available for RD {}", rd);
1786             if (callback != null) {
1787                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1788                 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1789                 Futures.addCallback(listenableFuture, callback);
1790             }
1791             return;
1792         }
1793         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1794         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1795             () -> {
1796                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1797                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1798                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1799                     for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1800                         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1801                         if (subnetRoute != null) {
1802                             long elanTag = subnetRoute.getElantag();
1803                             installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1804                             continue;
1805                         }
1806                         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1807                         if (routerInt != null) {
1808                             LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1809                                 rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1810                             installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
1811                                 routerInt.getIpAddress(),
1812                                 new MacAddress(routerInt.getMacAddress()), NwConstants.ADD_FLOW);
1813                             continue;
1814                         }
1815                         //Handle local flow creation for imports
1816                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1817                             java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1818                             if (optionalLabel.isPresent()) {
1819                                 List<String> nextHopList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1820                                 LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1821                                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1822                                     if (lri.getDpnId().equals(dpnId)) {
1823                                         createLocalFibEntry(vpnId, rd, vrfEntry);
1824                                         continue;
1825                                     }
1826                                 }
1827                             }
1828                         }
1829                         // Passing null as we don't know the dpn
1830                         // to which prefix is attached at this point
1831                         createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1832                     }
1833                     //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1834                     futures.add(tx.submit());
1835                 }
1836                 if (callback != null) {
1837                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1838                     Futures.addCallback(listenableFuture, callback);
1839                 }
1840                 return futures;
1841             });
1842     }
1843
1844
1845     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1846                                             final String localNextHopIp, final String remoteNextHopIp) {
1847         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1848             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1849         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1850         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1851         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1852         if (vrfTable.isPresent()) {
1853             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1854             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1855                 () -> {
1856                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1857                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1858                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1859                         vrfTable.get().getVrfEntry().stream()
1860                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1861                             .forEach(
1862                                 getConsumerForCreatingRemoteFib(dpnId, vpnId,
1863                                         rd, remoteNextHopIp, vrfTable,
1864                                         writeCfgTxn));
1865                         futures.add(writeCfgTxn.submit());
1866                     }
1867                     return futures;
1868                 });
1869         }
1870     }
1871
1872     public void populateInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1873                                             final String localNextHopIp, final String remoteNextHopIp) {
1874         LOG.trace("populateInternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1875             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1876         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1877         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1878         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1879         if (vrfTable.isPresent()) {
1880             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1881             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1882                 () -> {
1883                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1884                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1885                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1886                         // Handle Vpn Interface driven Routes only (i.e., STATIC and LOCAL)
1887                         vrfTable.get().getVrfEntry().stream()
1888                             .filter(vrfEntry -> {
1889                                 /* Ignore SubnetRoute entry */
1890                                 return (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1891                                         vrfEntry.getOrigin())));
1892                             } )
1893                             .forEach(getConsumerForCreatingRemoteFib(dpnId, vpnId,
1894                                        rd, remoteNextHopIp, vrfTable,
1895                                        writeCfgTxn));
1896                         futures.add(writeCfgTxn.submit());
1897                     }
1898                     return futures;
1899                 });
1900         }
1901     }
1902
1903     public void manageRemoteRouteOnDPN(final boolean action,
1904                                        final BigInteger localDpnId,
1905                                        final long vpnId,
1906                                        final String rd,
1907                                        final String destPrefix,
1908                                        final String destTepIp,
1909                                        final long label) {
1910         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1911
1912         if (vpnInstance == null) {
1913             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1914             return;
1915         }
1916         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1917         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + localDpnId.toString(),
1918             () -> {
1919                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1920                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1921                     WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1922                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1923                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1924                     if (vrfEntry == null) {
1925                         return futures;
1926                     }
1927                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1928                         action, localDpnId, vpnId, rd, destPrefix);
1929                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1930                     VrfEntry modVrfEntry;
1931                     if (routePathList == null || (routePathList.isEmpty())) {
1932                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1933                                 Collections.singletonList(destTepIp),
1934                                 RouteOrigin.value(vrfEntry.getOrigin()))
1935                                 .build();
1936                     } else {
1937                         modVrfEntry = vrfEntry;
1938                     }
1939
1940                     if (action == true) {
1941                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1942                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
1943                     } else {
1944                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1945                         deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
1946                     }
1947                     futures.add(writeTransaction.submit());
1948                 }
1949                 return futures;
1950             });
1951     }
1952
1953     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1954                                  final FutureCallback<List<Void>> callback) {
1955         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1956         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1957         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1958         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1959         if (vrfTable.isPresent()) {
1960             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1961             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1962                 () -> {
1963                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1964                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1965                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1966                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1967                                 /* Handle subnet routes here */
1968                             SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1969                             if (subnetRoute != null) {
1970                                 LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn",
1971                                         vrfEntry.getDestPrefix(),
1972                                         dpnId, rd);
1973                                 makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1974                                 java.util.Optional.ofNullable(vrfEntry.getRoutePaths()).ifPresent(routePaths -> {
1975                                     routePaths.stream().forEach(routePath -> {
1976                                         makeLFibTableEntry(dpnId, routePath.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY,
1977                                                 NwConstants.DEL_FLOW, tx);
1978                                         LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}",
1979                                                 routePath.getLabel(), rd,
1980                                                 vrfEntry.getDestPrefix());
1981                                     });
1982                                 });
1983                                 continue;
1984                             }
1985                             // ping responder for router interfaces
1986                             RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1987                             if (routerInt != null) {
1988                                 LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1989                                     rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1990                                 installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
1991                                     routerInt.getIpAddress(),
1992                                     new MacAddress(routerInt.getMacAddress()), NwConstants.DEL_FLOW);
1993                                 continue;
1994                             }
1995                             // Passing null as we don't know the dpn
1996                             // to which prefix is attached at this point
1997                             deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1998                         }
1999                         futures.add(tx.submit());
2000                         if (callback != null) {
2001                             ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
2002                             Futures.addCallback(listenableFuture, callback);
2003                         }
2004                     }
2005                     return futures;
2006                 });
2007         }
2008     }
2009
2010     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
2011                                            final String localNextHopIp, final String remoteNextHopIp) {
2012         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
2013                 + " localNexthopIp {} , remoteNexhtHopIp {}",
2014             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2015         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2016         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2017         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2018         if (vrfTable.isPresent()) {
2019             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2020             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2021                 () -> {
2022                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2023                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2024                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
2025                         vrfTable.get().getVrfEntry().stream()
2026                             .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
2027                             .forEach(getConsumerForDeletingRemoteFib(dpnId, vpnId, rd,
2028                                 remoteNextHopIp, vrfTable, writeCfgTxn));
2029                         futures.add(writeCfgTxn.submit());
2030                     }
2031                     return futures;
2032                 });
2033         }
2034     }
2035
2036     public void cleanUpInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
2037                                            final String localNextHopIp, final String remoteNextHopIp) {
2038         LOG.trace("cleanUpInternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
2039                 + " localNexthopIp {} , remoteNexhtHopIp {}",
2040             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2041         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2042         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2043         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2044         if (vrfTable.isPresent()) {
2045             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2046             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2047                 () -> {
2048                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2049                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2050                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
2051                         vrfTable.get().getVrfEntry().stream()
2052                             .filter(vrfEntry -> {
2053                                 /* Ignore SubnetRoute entry */
2054                                 return (FibHelper.isControllerManagedVpnInterfaceRoute(
2055                                         RouteOrigin.value(vrfEntry.getOrigin())));
2056                             })
2057                             .forEach(getConsumerForDeletingRemoteFib(dpnId, vpnId,
2058                                 rd, remoteNextHopIp, vrfTable, writeCfgTxn));
2059                         futures.add(writeCfgTxn.submit());
2060                     }
2061                     return futures;
2062                 });
2063         }
2064     }
2065
2066     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
2067         InstanceIdentifierBuilder<VrfTables> idBuilder =
2068             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
2069         InstanceIdentifier<VrfTables> id = idBuilder.build();
2070         return id;
2071     }
2072
2073     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
2074         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + label
2075                 + NwConstants.FLOWID_SEPARATOR + priority;
2076     }
2077
2078     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
2079         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + rd
2080                 + NwConstants.FLOWID_SEPARATOR + priority + NwConstants.FLOWID_SEPARATOR + destPrefix.getHostAddress();
2081     }
2082
2083     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
2084         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
2085                 .FLOWID_SEPARATOR + nextHop;
2086     }
2087
2088     protected List<AdjacencyResult> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
2089                                                      final VrfEntry vrfEntry, String rd) {
2090         List<AdjacencyResult> adjacencyList = new ArrayList<>();
2091         List<String> prefixIpList = new ArrayList<>();
2092         LOG.trace("resolveAdjacency called with remotedDpnId {}, vpnId{}, VrfEntry {}",
2093             remoteDpnId, vpnId, vrfEntry);
2094         try {
2095             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2096                 Routes extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
2097                 if (extraRoute == null) {
2098                     prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
2099                 } else {
2100                     prefixIpList = new ArrayList<>();
2101                     for (String extraRouteIp : extraRoute.getNexthopIpList()) {
2102                         prefixIpList.add(extraRouteIp + "/32");
2103                     }
2104                 }
2105             } else {
2106                 prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
2107             }
2108
2109             for (String prefixIp : prefixIpList) {
2110                 adjacencyList.addAll(vrfEntry.getRoutePaths().stream()
2111                         .map(routePath -> {
2112                             LOG.debug("NextHop IP for destination {} is {}", prefixIp,
2113                                     routePath.getNexthopAddress());
2114                             return nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
2115                                     prefixIp, routePath.getNexthopAddress());
2116                         })
2117                         .filter(adjacencyResult -> adjacencyResult != null && !adjacencyList.contains(adjacencyResult))
2118                         .distinct()
2119                         .collect(Collectors.toList()));
2120             }
2121         } catch (NullPointerException e) {
2122             LOG.trace("", e);
2123         }
2124         return adjacencyList;
2125     }
2126
2127     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
2128         InstanceIdentifier<VpnInstanceOpDataEntry> id =
2129             InstanceIdentifier.create(VpnInstanceOpData.class)
2130                 .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
2131         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
2132             FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
2133         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
2134     }
2135
2136     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2137         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
2138                 + tableMiss + FLOWID_PREFIX;
2139     }
2140
2141     /*
2142      * Install flow entry in protocol table to forward mpls
2143      * coming through gre tunnel to LFIB table.
2144      */
2145     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
2146         final BigInteger cookieProtocolTable = new BigInteger("1070000", 16);
2147         // Instruction to goto L3 InterfaceTable
2148         List<InstructionInfo> instructions = new ArrayList<>();
2149         instructions.add(new InstructionGotoTable(NwConstants.L3_LFIB_TABLE));
2150         List<MatchInfo> matches = new ArrayList<>();
2151         matches.add(MatchEthernetType.MPLS_UNICAST);
2152         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2153             getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2154                 NwConstants.L3_LFIB_TABLE),
2155             DEFAULT_FIB_FLOW_PRIORITY,
2156             "Protocol Table For LFIB",
2157             0, 0,
2158             cookieProtocolTable,
2159             matches, instructions);
2160
2161         if (addOrRemove == NwConstants.ADD_FLOW) {
2162             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2163             mdsalManager.installFlow(flowEntityToLfib);
2164         } else {
2165             mdsalManager.removeFlow(flowEntityToLfib);
2166         }
2167     }
2168
2169     public List<String> printFibEntries() {
2170         List<String> result = new ArrayList<>();
2171         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2172         result.add("-------------------------------------------------------------------");
2173         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2174         Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2175         if (fibEntries.isPresent()) {
2176             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2177             for (VrfTables vrfTable : vrfTables) {
2178                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2179                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
2180                     for (RoutePaths routePath : routePaths) {
2181                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2182                             vrfTable.getRouteDistinguisher(),
2183                             vrfEntry.getDestPrefix(), routePath.getNexthopAddress(),
2184                             routePath.getLabel(), vrfEntry.getOrigin()));
2185                     }
2186                     if (routePaths.isEmpty()) {
2187                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s",
2188                             vrfTable.getRouteDistinguisher(),
2189                             vrfEntry.getDestPrefix(), "local", vrfEntry.getOrigin()));
2190                     }
2191                 }
2192             }
2193         }
2194         return result;
2195     }
2196
2197
2198     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2199         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2200             .child(VrfTables.class, new VrfTablesKey(rd))
2201             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2202         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2203         if (vrfEntry.isPresent()) {
2204             return vrfEntry.get();
2205         }
2206         return null;
2207     }
2208
2209     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2210         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2211             .child(VrfTables.class, new VrfTablesKey(rd))
2212             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2213         return vrfEntryId;
2214     }
2215
2216     protected Boolean isIpv4Address(String ipAddress) {
2217         try {
2218             InetAddress address = InetAddress.getByName(ipAddress);
2219             if (address instanceof Inet4Address) {
2220                 return true;
2221             }
2222         } catch (UnknownHostException e) {
2223             LOG.warn("Invalid ip address {}", ipAddress, e);
2224             return false;
2225         }
2226         return false;
2227     }
2228
2229     protected Boolean installRouterFibEntries(final VrfEntry vrfEntry, final Collection<VpnToDpnList> vpnToDpnList,
2230                                               long vpnId, int addOrRemove) {
2231         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
2232         if (routerInt != null && vpnToDpnList != null) {
2233             String routerId = routerInt.getUuid();
2234             String macAddress = routerInt.getMacAddress();
2235             String ipValue = routerInt.getIpAddress();
2236             LOG.trace("createFibEntries - Router augmented vrfentry found for for router uuid:{}, ip:{}, mac:{}",
2237                 routerId, ipValue, macAddress);
2238             for (VpnToDpnList vpnDpn : vpnToDpnList) {
2239                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
2240                     installRouterFibEntry(vrfEntry, vpnDpn.getDpnId(), vpnId, routerId, ipValue,
2241                         new MacAddress(macAddress), addOrRemove);
2242                 }
2243             }
2244             return true;
2245         }
2246         return false;
2247     }
2248
2249     public void installRouterFibEntry(final VrfEntry vrfEntry, BigInteger dpnId, long vpnId, String routerUuid,
2250                                       String routerInternalIp, MacAddress routerMac, int addOrRemove) {
2251         String[] subSplit = routerInternalIp.split("/");
2252         if (!isIpv4Address(subSplit[0])) {
2253             // Ping responder using OpenFlow rules is only supported for IPv4, hence skipping.
2254             return;
2255         }
2256
2257         java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2258         if (!optionalLabel.isPresent()) {
2259             LOG.warn("Routes paths not present. Exiting installRouterFibEntry");
2260             return;
2261         }
2262         String addRemoveStr = (addOrRemove == NwConstants.ADD_FLOW) ? "ADD_FLOW" : "DELETE_FLOW";
2263         LOG.trace("{}: bulding Echo Flow entity for dpid:{}, router_ip:{}, vpnId:{}, subSplit:{} ", addRemoveStr,
2264             dpnId, routerInternalIp, vpnId, subSplit[0]);
2265
2266         List<MatchInfo> matches = new ArrayList<>();
2267
2268         matches.add(MatchIpProtocol.ICMP);
2269         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
2270         matches.add(new MatchIcmpv4((short) 8, (short) 0));
2271         matches.add(MatchEthernetType.IPV4);
2272         matches.add(new MatchIpv4Destination(subSplit[0], "32"));
2273
2274         List<ActionInfo> actionsInfos = new ArrayList<>();
2275
2276         // Set Eth Src and Eth Dst
2277         actionsInfos.add(new ActionMoveSourceDestinationEth());
2278         actionsInfos.add(new ActionSetFieldEthernetSource(routerMac));
2279
2280         // Move Ip Src to Ip Dst
2281         actionsInfos.add(new ActionMoveSourceDestinationIp());
2282         actionsInfos.add(new ActionSetSourceIp(subSplit[0], "32"));
2283
2284         // Set the ICMP type to 0 (echo reply)
2285         actionsInfos.add(new ActionSetIcmpType((short) 0));
2286
2287         actionsInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
2288
2289         actionsInfos.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2290
2291         List<InstructionInfo> instructions = new ArrayList<>();
2292
2293         instructions.add(new InstructionApplyActions(actionsInfos));
2294
2295         int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH;
2296         String flowRef = getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, optionalLabel.get(),
2297                 priority);
2298
2299         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef,
2300             0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions);
2301
2302         if (addOrRemove == NwConstants.ADD_FLOW) {
2303             mdsalManager.installFlow(flowEntity);
2304         } else {
2305             mdsalManager.removeFlow(flowEntity);
2306         }
2307     }
2308
2309     public void removeInterVPNLinkRouteFlows(final String interVpnLinkName,
2310                                              final boolean isVpnFirstEndPoint,
2311                                              final VrfEntry vrfEntry) {
2312         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
2313             && vrfEntry.getRoutePaths().size() == 1);
2314         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLinkName);
2315
2316         if (!interVpnLinkState.isPresent()) {
2317             LOG.warn("Could not find State for InterVpnLink {}", interVpnLinkName);
2318             return;
2319         }
2320
2321         List<BigInteger> targetDpns =
2322             isVpnFirstEndPoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
2323                 : interVpnLinkState.get().getSecondEndpointState().getDpId();
2324
2325         java.util.Optional<String> optionalNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
2326         java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2327
2328         // delete from FIB
2329         //
2330         optionalNextHop.ifPresent(nextHop -> {
2331             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2332             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2333             Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef))
2334                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
2335
2336             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}",
2337                     interVpnLinkName, flowRef);
2338
2339             for (BigInteger dpId : targetDpns) {
2340                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2341                         vrfEntry.getDestPrefix(), nextHop,
2342                         dpId, interVpnLinkName);
2343
2344                 mdsalManager.removeFlow(dpId, flow);
2345             }
2346         });
2347
2348         // delete from LFIB
2349         //
2350         optionalLabel.ifPresent(label -> {
2351             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2352
2353             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
2354             for (BigInteger dpId : targetDpns) {
2355                 LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
2356                         vrfEntry.getDestPrefix(), label,
2357                         dpId, interVpnLinkName);
2358                 makeLFibTableEntry(dpId, label, null /* no instructions */,
2359                         LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, tx);
2360             }
2361             tx.submit();
2362         });
2363     }
2364
2365     private Consumer<? super VrfEntry> getConsumerForCreatingRemoteFib(
2366             final BigInteger dpnId, final long vpnId, final String rd,
2367             final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
2368             WriteTransaction writeCfgTxn) {
2369         return vrfEntry -> vrfEntry.getRoutePaths().stream()
2370                 .filter(routes -> !routes.getNexthopAddress().isEmpty()
2371                         && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
2372                 .findFirst()
2373                 .ifPresent(routes -> {
2374                     LOG.trace("creating remote FIB entry for prefix {} rd {} on Dpn {}",
2375                             vrfEntry.getDestPrefix(), rd, dpnId);
2376                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeCfgTxn);
2377                 });
2378     }
2379
2380     private Consumer<? super VrfEntry> getConsumerForDeletingRemoteFib(
2381             final BigInteger dpnId, final long vpnId, final String rd,
2382             final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
2383             WriteTransaction writeCfgTxn) {
2384         return vrfEntry -> vrfEntry.getRoutePaths().stream()
2385                 .filter(routes -> !routes.getNexthopAddress().isEmpty()
2386                         && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
2387                 .findFirst()
2388                 .ifPresent(routes -> {
2389                     LOG.trace(" deleting remote FIB entry {}", vrfEntry);
2390                     deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeCfgTxn);
2391                 });
2392     }
2393
2394     private boolean isPrefixAndNextHopPresentInLri(String prefix,
2395             List<String> nextHopAddressList, LabelRouteInfo lri) {
2396         return lri != null && lri.getPrefix().equals(prefix)
2397                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
2398     }
2399 }