vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static java.util.stream.Collectors.toList;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17
  18 import java.math.BigInteger;
  19 import java.net.Inet4Address;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.List;
  27 import java.util.concurrent.BlockingQueue;
  28 import java.util.concurrent.Callable;
  29 import java.util.concurrent.LinkedBlockingQueue;
  30 import java.util.function.Consumer;
  31
  32 import javax.annotation.PostConstruct;
  33 import javax.inject.Inject;
  34 import javax.inject.Singleton;
  35 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  36 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  37 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  38 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  39 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  40 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  41 import org.opendaylight.genius.mdsalutil.ActionInfo;
  42 import org.opendaylight.genius.mdsalutil.FlowEntity;
  43 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  44 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  45 import org.opendaylight.genius.mdsalutil.MatchInfo;
  46 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  47 import org.opendaylight.genius.mdsalutil.NwConstants;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationEth;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationIp;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionSetIcmpType;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionSetSourceIp;
  62 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  63 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  64 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  65 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  67 import org.opendaylight.genius.mdsalutil.matches.MatchIcmpv4;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchIpv6Destination;
  71 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  72 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  73 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  74 import org.opendaylight.genius.utils.ServiceIndex;
  75 import org.opendaylight.genius.utils.batching.ActionableResource;
  76 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  77 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  78 import org.opendaylight.genius.utils.batching.ResourceHandler;
  79 import org.opendaylight.genius.utils.batching.SubTransaction;
  80 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
  81 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  82 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  83 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  84 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  85 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  86 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  87 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  88 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  89 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroutes;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.Vpn;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.VpnKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutes;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutesKey;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.RoutesKey;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 141 import org.opendaylight.yangtools.yang.binding.DataObject;
 142 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 143 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 144 import org.slf4j.Logger;
 145 import org.slf4j.LoggerFactory;
 146
 147 @Singleton
 148 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener>
 149     implements AutoCloseable, ResourceHandler {
 150
 151     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 152     private static final String FLOWID_PREFIX = "L3.";
 153     private static final int BATCH_SIZE = 1000;
 154     private static final int PERIODICITY = 500;
 155     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 156     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 157     private static final int LFIB_INTERVPN_PRIORITY = 15;
 158     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 159
 160     private final DataBroker dataBroker;
 161     private final IMdsalApiManager mdsalManager;
 162     private final NexthopManager nextHopManager;
 163     private final OdlInterfaceRpcService interfaceManager;
 164     private final IdManagerService idManager;
 165     private final IVpnLinkService ivpnLinkService;
 166     List<SubTransaction> transactionObjects;
 167
 168     private static Integer batchSize;
 169     private static Integer batchInterval;
 170
 171     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 172     private final ResourceBatchingManager resourceBatchingManager;
 173
 174     @Inject
 175     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 176                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
 177                             final IdManagerService idManager, final IVpnLinkService ivpnLinkService) {
 178         super(VrfEntry.class, VrfEntryListener.class);
 179         this.dataBroker = dataBroker;
 180         this.mdsalManager = mdsalApiManager;
 181         this.nextHopManager = nexthopManager;
 182         this.interfaceManager = interfaceManager;
 183         this.idManager = idManager;
 184         this.ivpnLinkService = ivpnLinkService;
 185
 186         batchSize = Integer.getInteger("batch.size", BATCH_SIZE);
 187         batchInterval = Integer.getInteger("batch.wait.time", PERIODICITY);
 188         resourceBatchingManager = ResourceBatchingManager.getInstance();
 189         resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
 190         transactionObjects = new ArrayList<>();
 191     }
 192
 193     @Override
 194     @PostConstruct
 195     public void init() {
 196         LOG.info("{} init", getClass().getSimpleName());
 197         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 198     }
 199
 200     @Override
 201     protected VrfEntryListener getDataTreeChangeListener() {
 202         return VrfEntryListener.this;
 203     }
 204
 205     @Override
 206     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 207         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 208     }
 209
 210     @Override
 211     public DataBroker getResourceBroker() {
 212         return dataBroker;
 213     }
 214
 215     public NexthopManager getNextHopManager() {
 216         return this.nextHopManager;
 217     }
 218
 219     @Override
 220     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 221         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 222         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 223         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 224                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 225         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 226             LOG.info("EVPN flows need to be programmed.");
 227             EVPNVrfEntryProcessor evpnVrfEntryProcessor = new EVPNVrfEntryProcessor(identifier,
 228                     vrfEntry, dataBroker, this);
 229             evpnVrfEntryProcessor.installFlows();
 230         } else {
 231             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 232                 createFibEntries(identifier, vrfEntry);
 233             } else {
 234                 ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
 235                 actResource.setAction(ActionableResource.CREATE);
 236                 actResource.setInstanceIdentifier(identifier);
 237                 actResource.setInstance(vrfEntry);
 238                 vrfEntryBufferQ.add(actResource);
 239             }
 240         }
 241         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 242         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 243             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 244     }
 245
 246     @Override
 247     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 248         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 249         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 250         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 251                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 252         if (vrfEntry.getEncapType().equals(VrfEntry.EncapType.Vxlan)) {
 253             LOG.info("EVPN flows to be deleted");
 254             EVPNVrfEntryProcessor evpnVrfEntryProcessor = new EVPNVrfEntryProcessor(identifier, vrfEntry,
 255                     dataBroker, this);
 256             evpnVrfEntryProcessor.removeFlows();
 257         } else {
 258             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 259                 deleteFibEntries(identifier, vrfEntry);
 260             } else {
 261                 ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
 262                 actResource.setAction(ActionableResource.DELETE);
 263                 actResource.setInstanceIdentifier(identifier);
 264                 actResource.setInstance(vrfEntry);
 265                 vrfEntryBufferQ.add(actResource);
 266             }
 267         }
 268         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 269         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 270             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 271     }
 272
 273     @Override
 274     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 275         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 276
 277         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 278         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
 279         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {}",
 280             rd, update.getDestPrefix(), update.getRoutePaths());
 281         // Handle BGP Routes first
 282         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 283             ActionableResource actResource = new ActionableResourceImpl(rd + update.getDestPrefix());
 284             actResource.setAction(ActionableResource.UPDATE);
 285             actResource.setInstanceIdentifier(identifier);
 286             actResource.setInstance(update);
 287             actResource.setOldInstance(original);
 288             vrfEntryBufferQ.add(actResource);
 289             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 290                 rd, update.getDestPrefix(), update.getRoutePaths());
 291             return;
 292         }
 293
 294         // Handle Vpn Interface driven Routes next (ie., STATIC and LOCAL)
 295         if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(update.getOrigin()))) {
 296             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 297             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 298             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 299
 300             // If original VRF Entry had nexthop null , but update VRF Entry
 301             // has nexthop , route needs to be created on remote Dpns
 302             if (((originalRoutePath == null) || (originalRoutePath.isEmpty())
 303                 && (updateRoutePath != null) && (!updateRoutePath.isEmpty()))) {
 304                 // TODO(vivek): Though ugly, Not handling this code now, as each
 305                 // tep add event will invoke flow addition
 306                 LOG.trace("Original VRF entry NH is null for destprefix {}. This event is IGNORED here.",
 307                     update.getDestPrefix());
 308                 return;
 309             }
 310
 311             // If original VRF Entry had valid nexthop , but update VRF Entry
 312             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 313             if (((updateRoutePath == null) || (updateRoutePath.isEmpty())
 314                 && (originalRoutePath != null) && (!originalRoutePath.isEmpty()))) {
 315                 LOG.trace("Original VRF entry had valid NH for destprefix {}. This event is IGNORED here.",
 316                     update.getDestPrefix());
 317                 return;
 318             }
 319             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 320             List<String> nextHopsRemoved = FibUtil.getNextHopListFromRoutePaths(original);
 321             nextHopsRemoved.removeAll(FibUtil.getNextHopListFromRoutePaths(update));
 322             WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 323             nextHopsRemoved.parallelStream()
 324                     .forEach(nextHopRemoved -> FibUtil.updateUsedRdAndVpnToExtraRoute(
 325                              writeOperTxn, dataBroker, nextHopRemoved, rd,
 326                              update.getDestPrefix()));
 327             writeOperTxn.submit();
 328             createFibEntries(identifier, update);
 329             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 330                 rd, update.getDestPrefix(), update.getRoutePaths());
 331             return;
 332         }
 333
 334         /* Handl all other route origins */
 335         createFibEntries(identifier, update);
 336
 337         LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 338             rd, update.getDestPrefix(), update.getRoutePaths());
 339     }
 340
 341     @Override
 342     public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 343                        Object original, Object update, List<SubTransaction> transactionObjects) {
 344         this.transactionObjects = transactionObjects;
 345         if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 346             createFibEntries(tx, identifier, (VrfEntry) update);
 347         }
 348     }
 349
 350     @Override
 351     public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 352                        Object vrfEntry, List<SubTransaction> transactionObjects) {
 353         this.transactionObjects = transactionObjects;
 354         if (vrfEntry instanceof VrfEntry) {
 355             createFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 356         }
 357     }
 358
 359     @Override
 360     public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 361                        Object vrfEntry, List<SubTransaction> transactionObjects) {
 362         this.transactionObjects = transactionObjects;
 363         if (vrfEntry instanceof VrfEntry) {
 364             deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 365         }
 366     }
 367
 368     @Override
 369     public int getBatchSize() {
 370         return batchSize;
 371     }
 372
 373     @Override
 374     public int getBatchInterval() {
 375         return batchInterval;
 376     }
 377
 378     @Override
 379     public LogicalDatastoreType getDatastoreType() {
 380         return LogicalDatastoreType.CONFIGURATION;
 381     }
 382
 383     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 384         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 385
 386         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 387         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 388         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 389                 + " has null vpnId!");
 390         final Collection<VpnToDpnList> vpnToDpnList;
 391         if (vrfEntry.getParentVpnRd() != null
 392                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 393             VpnInstanceOpDataEntry parentVpnInstance = getVpnInstance(vrfEntry.getParentVpnRd());
 394             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 395                 vpnInstance.getVpnToDpnList();
 396         } else {
 397             vpnToDpnList = vpnInstance.getVpnToDpnList();
 398         }
 399
 400         final Long vpnId = vpnInstance.getVpnId();
 401         final String rd = vrfTableKey.getRouteDistinguisher();
 402         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 403         if (subnetRoute != null) {
 404             final long elanTag = subnetRoute.getElantag();
 405             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 406                     rd, vrfEntry.getDestPrefix(), elanTag);
 407             if (vpnToDpnList != null) {
 408                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 409                 dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(), () -> {
 410                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 411                     for (final VpnToDpnList curDpn : vpnToDpnList) {
 412                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 413                             installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 414                         }
 415                     }
 416                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 417                     futures.add(tx.submit());
 418                     return futures;
 419                 });
 420             }
 421             return;
 422         }
 423         // ping responder for router interfaces
 424         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnId, NwConstants.ADD_FLOW)) {
 425             return;
 426         }
 427
 428         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 429         if (!localDpnIdList.isEmpty()) {
 430             if (vpnToDpnList != null) {
 431                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 432                 dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(), () -> {
 433                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 434                     for (VpnToDpnList vpnDpn : vpnToDpnList) {
 435                         if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 436                             if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 437                                 createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry,
 438                                         tx);
 439                             }
 440                         }
 441                     }
 442                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 443                     futures.add(tx.submit());
 444                     return futures;
 445                 });
 446             }
 447         }
 448
 449         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 450         if (optVpnUuid.isPresent()) {
 451             Optional<InterVpnLinkDataComposite> optInterVpnLink =
 452                 InterVpnLinkCache.getInterVpnLinkByVpnId(optVpnUuid.get());
 453             LOG.debug("InterVpnLink {} found in Cache: {}", optVpnUuid.get(), optInterVpnLink.isPresent());
 454             if (optInterVpnLink.isPresent()) {
 455                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
 456                 String vpnUuid = optVpnUuid.get();
 457                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 458                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 459                         // This is an static route that points to the other endpoint of an InterVpnLink
 460                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 461                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 462                         installInterVpnRouteInLFib(rd, vrfEntry);
 463                     }
 464                 });
 465             }
 466         }
 467     }
 468
 469     /*
 470       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 471       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 472       provided by ResourceBatchingManager
 473      */
 474     private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
 475                                   final VrfEntry vrfEntry) {
 476         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 477
 478         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 479         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 480         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 481             + " has null vpnId!");
 482
 483         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 484         if (vpnToDpnList != null) {
 485             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 486                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 487                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 488                 }
 489             }
 490         }
 491     }
 492
 493     void refreshFibTables(String rd, String prefix) {
 494         InstanceIdentifier<VrfEntry> vrfEntryId =
 495                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 496                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 497         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 498         if (vrfEntry.isPresent()) {
 499             createFibEntries(vrfEntryId, vrfEntry.get());
 500         }
 501     }
 502
 503     // FIXME: Refactoring needed here.
 504     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 505     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 506                                    int addOrRemove) {
 507         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 508         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 509
 510         String prefix = vrfEntry.getDestPrefix();
 511         List<String> nextHopsList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 512         // Label is used only for logging in subsequent method calls.
 513         //TODO : This label is not needed here. Can be removed. Hence using a default value.
 514         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(0L);
 515         String rd = vrfTableKey.getRouteDistinguisher();
 516         LOG.trace("leakRouteIfNeeded: srcVpnRd={}  prefix={}  nhList={}  label={}", rd, prefix, nextHopsList, label);
 517
 518         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 519         if (vpnInstance == null) {
 520             LOG.error("VPN Instance not available for route with prefix {} label {} nextHop {} RD {}. Returning...",
 521                       prefix, label, nextHopsList, rd);
 522             return;
 523         }
 524         String vpnUuid = vpnInstance.getVpnInstanceName();
 525         if (vpnUuid == null || vpnUuid.isEmpty()) {
 526             LOG.warn("Could not find suitable VPN UUID for rd={}.  vrfEntry=[prefix={}  nhList={}  label={}]",
 527                      rd, prefix, nextHopsList, label);
 528             return;
 529         }
 530
 531         ivpnLinkService.leakRouteIfNeeded(vpnUuid, prefix, nextHopsList, label.intValue(),
 532                                           RouteOrigin.value(vrfEntry.getOrigin()), addOrRemove);
 533     }
 534
 535     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 536         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 537         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 538         prefixBuilder.setDpnId(lri.getDpnId());
 539         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 540         prefixBuilder.setIpAddress(lri.getPrefix());
 541         // Increment the refCount here
 542         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 543             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 544         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 545         if (!isPresentInList) {
 546             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 547             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 548             vpnInstanceNames.add(vpnInstanceName);
 549             builder.setVpnInstanceList(vpnInstanceNames);
 550             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(),
 551                 FibUtil.DEFAULT_CALLBACK);
 552         } else {
 553             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 554         }
 555         return prefixBuilder.build();
 556     }
 557
 558     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 559                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 560         Boolean wrTxPresent = true;
 561         if (tx == null) {
 562             wrTxPresent = false;
 563             tx = dataBroker.newWriteOnlyTransaction();
 564         }
 565         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 566             List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 567             synchronized (label.toString().intern()) {
 568                 LabelRouteInfo lri = getLabelRouteInfo(label);
 569                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 570
 571                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 572                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 573                                 FibUtil.getVpnInstanceOpData(dataBroker, rd);
 574                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 575                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 576                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 577                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 578                             }
 579                         }
 580                     }
 581                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 582                             label, lri.getVpnInterfaceName(), lri.getDpnId());
 583                 }
 584             }
 585         });
 586         final List<InstructionInfo> instructions = new ArrayList<>();
 587         BigInteger subnetRouteMeta = ((BigInteger.valueOf(elanTag)).shiftLeft(24))
 588             .or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 589         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 590         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 591         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 592
 593         if (vrfEntry.getRoutePaths() != null) {
 594             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 595                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 596                     List<ActionInfo> actionsInfos = new ArrayList<>();
 597                     // reinitialize instructions list for LFIB Table
 598                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 599
 600                     actionsInfos.add(new ActionPopMpls());
 601                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 602                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 603                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 604                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 605
 606                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 607                             NwConstants.ADD_FLOW, tx);
 608                 }
 609             }
 610         }
 611         if (!wrTxPresent) {
 612             tx.submit();
 613         }
 614     }
 615
 616     /*
 617      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 618      * LportDispatcher table (via table 80)
 619      */
 620     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 621         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 622         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 623         // packet is commuted from Vpn2 to Vpn1.
 624         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
 625         if (!vpnNameOpc.isPresent()) {
 626             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 627             return;
 628         }
 629
 630         String vpnName = vpnNameOpc.get();
 631         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
 632         boolean interVpnLinkFound = false;
 633         for (InterVpnLink interVpnLink : interVpnLinks) {
 634             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 635             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 636                 && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 637             if (vpnIs1stEndpoint || vpnIs2ndEndpoint) {
 638                 interVpnLinkFound = true;
 639
 640                 Optional<InterVpnLinkState> vpnLinkState =
 641                     FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 642                 if (!vpnLinkState.isPresent()
 643                     || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active)) {
 644                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 645                         interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 646                         interVpnLink.getSecondEndpoint().getVpnUuid().getValue());
 647                     return;
 648                 }
 649
 650                 List<BigInteger> targetDpns = vpnIs1stEndpoint ? vpnLinkState.get().getFirstEndpointState().getDpId()
 651                     : vpnLinkState.get().getSecondEndpointState().getDpId();
 652                 Long lportTag = vpnIs1stEndpoint ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 653                     : vpnLinkState.get().getFirstEndpointState().getLportTag();
 654
 655                 LOG.trace("Installing flow in LFIB table for interVpnLink {}", interVpnLink.getName());
 656
 657                 for (BigInteger dpId : targetDpns) {
 658                     List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls());
 659
 660                     List<InstructionInfo> instructions =
 661                         Arrays.asList(new InstructionApplyActions(actionsInfos),
 662                             new InstructionWriteMetadata(
 663                                 MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 664                                     ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 665                                         NwConstants.L3VPN_SERVICE_INDEX)),
 666                                 MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 667                             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 668
 669                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(interVpnRoutePathLabel -> {
 670                         List<String> interVpnNextHopList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 671                         LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for "
 672                                 + "InterVpnLink {} in LFIB",
 673                                 vrfEntry.getDestPrefix(), interVpnRoutePathLabel, interVpnNextHopList,
 674                                 dpId, interVpnLink.getName());
 675
 676                         makeLFibTableEntry(dpId, interVpnRoutePathLabel, instructions, LFIB_INTERVPN_PRIORITY,
 677                                 NwConstants.ADD_FLOW, null);
 678                     });
 679                 }
 680
 681                 break;
 682             }
 683         }
 684
 685         if (!interVpnLinkFound) {
 686             LOG.warn("VrfEntry=[prefix={} route-paths={}] for VPN {} has origin INTERVPN but "
 687                 + "no InterVpnLink could be found",
 688                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), rd);
 689         }
 690     }
 691
 692
 693     /*
 694      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 695      */
 696     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 697                                                final VrfEntry vrfEntry, long vpnTag) {
 698         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 699         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 700             && vrfEntry.getRoutePaths().size() == 1);
 701         String destination = vrfEntry.getDestPrefix();
 702         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 703         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 704
 705         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 706         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 707         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 708         if (interVpnLink.getState().or(State.Error) != State.Active) {
 709             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 710                 destination, nextHop, interVpnLinkName);
 711             return;
 712         }
 713
 714         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 715         if (!optOtherEndpointLportTag.isPresent()) {
 716             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 717                 vpnUuid, interVpnLinkName);
 718             return;
 719         }
 720
 721         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 722         if (targetDpns.isEmpty()) {
 723             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 724                 vpnUuid, interVpnLinkName);
 725             return;
 726         }
 727
 728         String[] values = destination.split("/");
 729         String destPrefixIpAddress = values[0];
 730         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 731
 732         List<MatchInfo> matches = new ArrayList<>();
 733         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 734         matches.add(MatchEthernetType.IPV4);
 735
 736         if (prefixLength != 0) {
 737             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 738         }
 739
 740         List<Instruction> instructions =
 741             Arrays.asList(new InstructionWriteMetadata(
 742                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 743                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 744                             .L3VPN_SERVICE_INDEX)),
 745                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 746                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 747
 748         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 749         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 750         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 751             COOKIE_VM_FIB_TABLE, matches, instructions);
 752
 753         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 754             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 755
 756         for (BigInteger dpId : targetDpns) {
 757
 758             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 759                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 760                 dpId, interVpnLink.getInterVpnLinkName());
 761
 762             mdsalManager.installFlow(dpId, flowEntity);
 763         }
 764     }
 765
 766
 767     // TODO Clean up the exception handling
 768     @SuppressWarnings("checkstyle:IllegalCatch")
 769     private <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 770                                                     InstanceIdentifier<T> path) {
 771
 772         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 773
 774         Optional<T> result = Optional.absent();
 775         try {
 776             result = tx.read(datastoreType, path).get();
 777         } catch (Exception e) {
 778             throw new RuntimeException(e);
 779         }
 780
 781         return result;
 782     }
 783
 784     private List<BigInteger> getDpnIdForPrefix(DataBroker broker, Long vpnId, String rd, VrfEntry vrfEntry) {
 785         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 786         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, vrfEntry.getDestPrefix());
 787
 788         if (localNextHopInfo == null) {
 789             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 790             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
 791                     FibUtil.getVpnNameFromId(dataBroker, vpnId), rd, vrfEntry.getDestPrefix());
 792             if (extraRouteOptional.isPresent()) {
 793                 Routes extraRoute = extraRouteOptional.get();
 794                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 795                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 796                     if (nextHopIp != null) {
 797                         localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, nextHopIp
 798                                 + NwConstants.IPV4PREFIX);
 799                         if (localNextHopInfo != null) {
 800                             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 801                         }
 802                     }
 803                 }
 804             }
 805         } else {
 806             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 807         }
 808
 809         return returnLocalDpnId;
 810     }
 811
 812     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 813         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 814         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
 815         String localNextHopIP = vrfEntry.getDestPrefix();
 816         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
 817         if (localNextHopInfo == null) {
 818             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
 819             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 820                     vpnName, usedRds, localNextHopIP);
 821             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 822             vpnExtraRoutes.stream().forEach(extraRoute -> {
 823                 Prefixes localNextHopInfoLocal = FibUtil.getPrefixToInterface(dataBroker,
 824                         vpnId, extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX);
 825                 BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfoLocal, vrfEntry.getDestPrefix(),
 826                         vpnId, rd, vrfEntry, vpnId, extraRoute, vpnExtraRoutes);
 827                 returnLocalDpnId.add(dpnId);
 828             });
 829             if (localNextHopInfo == null) {
 830             /* imported routes case */
 831                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 832                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 833                     if (optionalLabel.isPresent()) {
 834                         Long label = optionalLabel.get();
 835                         List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 836                         synchronized (label.toString().intern()) {
 837                             LabelRouteInfo lri = getLabelRouteInfo(label);
 838                             if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 839                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 840                                         FibUtil.getVpnInstanceOpData(dataBroker, rd);
 841                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
 842                                     String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 843                                     if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 844                                         localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 845                                         localNextHopIP = lri.getPrefix();
 846                                     } else {
 847                                         localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 848                                         localNextHopIP = lri.getPrefix();
 849                                     }
 850                                 }
 851                                 if (localNextHopInfo != null) {
 852                                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 853                                             label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 854                                     if (vpnExtraRoutes == null || vpnExtraRoutes.isEmpty()) {
 855                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 856                                                 vpnId, rd, vrfEntry, lri.getParentVpnid(), null, vpnExtraRoutes);
 857                                         returnLocalDpnId.add(dpnId);
 858                                     } else {
 859                                         for (Routes extraRoutes : vpnExtraRoutes) {
 860                                             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo,
 861                                                     localNextHopIP,
 862                                                     vpnId, rd, vrfEntry, lri.getParentVpnid(),
 863                                                     extraRoutes, vpnExtraRoutes);
 864                                             returnLocalDpnId.add(dpnId);
 865                                         }
 866                                     }
 867                                 }
 868                             }
 869                         }
 870                     }
 871                 }
 872             }
 873             if (returnLocalDpnId.isEmpty()) {
 874                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 875             }
 876         } else {
 877             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 878                     rd, vrfEntry, vpnId, /*routes*/ null, /*vpnExtraRoutes*/ null);
 879             returnLocalDpnId.add(dpnId);
 880         }
 881         return returnLocalDpnId;
 882     }
 883
 884     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 885                                                 final Long vpnId, final String rd,
 886                                                 final VrfEntry vrfEntry, Long parentVpnId,
 887                                                 Routes routes, List<Routes> vpnExtraRoutes) {
 888         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
 889         if (localNextHopInfo != null) {
 890             long groupId;
 891             long localGroupId;
 892             final BigInteger dpnId = localNextHopInfo.getDpnId();
 893             if (Boolean.TRUE.equals(localNextHopInfo.isNatPrefix())) {
 894                 LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip local dpn {} FIB processing",
 895                         vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 896                 return dpnId;
 897             }
 898             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 899             if (routes != null) {
 900                 groupId = nextHopManager.createNextHopGroups(parentVpnId, rd, dpnId, vrfEntry, routes,
 901                         vpnExtraRoutes);
 902                 localGroupId = nextHopManager.getLocalNextHopGroup(parentVpnId,
 903                         routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX);
 904             } else {
 905                 groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId,
 906                         localNextHopInfo.getVpnInterfaceName(), localNextHopIP, vrfEntry.getDestPrefix(),
 907                         vrfEntry.getGatewayMacAddress(), jobKey);
 908                 localGroupId = groupId;
 909             }
 910             if (groupId == FibConstants.INVALID_GROUP_ID) {
 911                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 912                     vrfEntry.getDestPrefix(), rd, localNextHopInfo.getVpnInterfaceName(), dpnId.toString());
 913                 return BigInteger.ZERO;
 914             }
 915             final List<InstructionInfo> instructions = Collections.singletonList(
 916                 new InstructionApplyActions(
 917                     Collections.singletonList(new ActionGroup(groupId))));
 918             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 919                 new InstructionApplyActions(
 920                     Arrays.asList(new ActionPopMpls(), new ActionGroup(groupId))));
 921             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 922             List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 923             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 924                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 925                     dpnId, localNextHopInfo.getVpnInterfaceName(), optLabel);
 926             } else {
 927                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. "
 928                     + "LFib and Terminating table entries will not be created.",
 929                     rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 930             }
 931             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 932
 933             dataStoreCoordinator.enqueueJob(jobKey,
 934                 () -> {
 935                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 936                     makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 937                     optLabel.ifPresent(label -> {
 938                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 939                             makeLFibTableEntry(dpnId, label, lfibinstructions,
 940                                     DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 941                             makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 942                         }
 943                     });
 944
 945                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 946                     futures.add(tx.submit());
 947                     return futures;
 948                 });
 949             return dpnId;
 950         }
 951         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}",
 952                 vrfEntry.getDestPrefix(), rd, vpnName);
 953         return BigInteger.ZERO;
 954     }
 955
 956     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId) {
 957         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 958         return FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id).isPresent();
 959     }
 960
 961     private LabelRouteInfo getLabelRouteInfo(Long label) {
 962         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 963             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 964         Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 965         if (opResult.isPresent()) {
 966             return opResult.get();
 967         }
 968         return null;
 969     }
 970
 971     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
 972         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 973         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 974             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 975         if (lri == null) {
 976             return true;
 977         }
 978         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 979             ? lri.getVpnInstanceList() : new ArrayList<>();
 980         if (vpnInstancesList.contains(vpnInstanceName)) {
 981             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 982             vpnInstancesList.remove(vpnInstanceName);
 983         }
 984         if (vpnInstancesList.size() == 0) {
 985             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 986             if (tx != null) {
 987                 tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
 988             } else {
 989                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 990             }
 991             return true;
 992         } else {
 993             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 994             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 995             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(),
 996                 FibUtil.DEFAULT_CALLBACK);
 997         }
 998         return false;
 999     }
1000
1001     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
1002                                       WriteTransaction tx) {
1003         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
1004
1005         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
1006
1007         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
1008             dpId, label, groupId);
1009     }
1010
1011     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
1012                                                 WriteTransaction tx) {
1013         List<MatchInfo> mkMatches = new ArrayList<>();
1014
1015         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
1016             destDpId, label, actionsInfos);
1017
1018         // Matching metadata
1019         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
1020         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1021
1022         List<InstructionInfo> mkInstructions = new ArrayList<>();
1023         mkInstructions.add(new InstructionApplyActions(actionsInfos));
1024
1025         FlowEntity terminatingServiceTableFlowEntity =
1026             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1027             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
1028                 String.format("%s:%d", "TST Flow Entry ", label),
1029             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
1030
1031         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1032
1033         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1034
1035         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1036         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1037             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1038             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1039             .child(Flow.class, flowKey).build();
1040         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(), true);
1041     }
1042
1043     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
1044         FlowEntity flowEntity;
1045         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1046         List<MatchInfo> mkMatches = new ArrayList<>();
1047         // Matching metadata
1048         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1049         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1050             NwConstants.INTERNAL_TUNNEL_TABLE,
1051             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1052             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1053             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1054         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
1055         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1056         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1057             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1058             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1059
1060         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1061         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1062     }
1063
1064     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1065         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1066         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1067         String localNextHopIP = vrfEntry.getDestPrefix();
1068         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
1069
1070         if (localNextHopInfo == null) {
1071             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1072             if (usedRds.size() > 1) {
1073                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1074                         vrfEntry.getDestPrefix(), vpnName, rd);
1075                 return returnLocalDpnId;
1076             }
1077             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1078             //in the vpn
1079             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1080                     vpnName, rd, vrfEntry.getDestPrefix());
1081             if (extraRouteOptional.isPresent()) {
1082                 Routes extraRoute = extraRouteOptional.get();
1083                 localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId,
1084                         extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX);
1085                 if (localNextHopInfo != null) {
1086                     BigInteger dpnId = localNextHopInfo.getDpnId();
1087                     if (!dpnId.equals(BigInteger.ZERO)) {
1088                         nextHopManager.setupLoadBalancingNextHop(vpnId, dpnId,
1089                                 vrfEntry.getDestPrefix(), /*listBucketInfo*/ null, /*remove*/ false);
1090                         returnLocalDpnId.add(dpnId);
1091                     }
1092                 } else {
1093                     LOG.error("localNextHopInfo unavailable.");
1094                 }
1095             }
1096
1097             if (localNextHopInfo == null) {
1098                 /* Imported VRF entry */
1099                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1100                 if (optionalLabel.isPresent()) {
1101                     Long label = optionalLabel.get();
1102                     List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1103                     LabelRouteInfo lri = getLabelRouteInfo(label);
1104                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1105                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1106                         prefixBuilder.setDpnId(lri.getDpnId());
1107                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), localNextHopIP,
1108                                 vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1109                         if (!dpnId.equals(BigInteger.ZERO)) {
1110                             returnLocalDpnId.add(dpnId);
1111                         }
1112                     }
1113                 }
1114             }
1115
1116         } else {
1117             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1118                 vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1119             if (!dpnId.equals(BigInteger.ZERO)) {
1120                 returnLocalDpnId.add(dpnId);
1121             }
1122         }
1123
1124         return returnLocalDpnId;
1125     }
1126
1127     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1128                                                 final Long vpnId, final String rd,
1129                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
1130         if (localNextHopInfo != null) {
1131             final BigInteger dpnId = localNextHopInfo.getDpnId();
1132             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1133             dataStoreCoordinator.enqueueJob("FIB-" + vpnId.toString() + "-"
1134                     + dpnId.toString() + "-" + vrfEntry.getDestPrefix(),
1135                 () -> {
1136                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1137                     makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
1138                         NwConstants.DEL_FLOW, tx);
1139                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1140                         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1141                             makeLFibTableEntry(dpnId, label, null /* instructions */,
1142                                     DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1143                             removeTunnelTableEntry(dpnId, label, tx);
1144                         });
1145                     }
1146                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1147                     futures.add(tx.submit());
1148                     return futures;
1149                 });
1150             //TODO: verify below adjacency call need to be optimized (?)
1151             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1152             return dpnId;
1153         }
1154         return BigInteger.ZERO;
1155     }
1156
1157     static  InstanceIdentifier<Routes> getVpnToExtrarouteIdentifier(String vpnName, String vrfId,
1158             String ipPrefix) {
1159         return InstanceIdentifier.builder(VpnToExtraroutes.class)
1160                 .child(Vpn.class, new VpnKey(vpnName)).child(ExtraRoutes.class,
1161                         new ExtraRoutesKey(vrfId)).child(Routes.class, new RoutesKey(ipPrefix)).build();
1162     }
1163
1164     public Routes getVpnToExtraroute(Long vpnId, String vpnRd, String destPrefix) {
1165         String optVpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
1166         if (optVpnName != null) {
1167             InstanceIdentifier<Routes> vpnExtraRoutesId = getVpnToExtrarouteIdentifier(
1168                     optVpnName, vpnRd, destPrefix);
1169             return FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, vpnExtraRoutesId).orNull();
1170         }
1171         return null;
1172     }
1173
1174     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1175             final VrfEntry vrfEntry, WriteTransaction tx) {
1176         Boolean wrTxPresent = true;
1177         if (tx == null) {
1178             wrTxPresent = false;
1179             tx = dataBroker.newWriteOnlyTransaction();
1180         }
1181         String rd = vrfTableKey.getRouteDistinguisher();
1182         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
1183         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1184                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1185
1186         List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1187         if (adjacencyResults == null || adjacencyResults.isEmpty()) {
1188             LOG.error("Could not get interface for route-paths: {} in vpn {}",
1189                     vrfEntry.getRoutePaths(), rd);
1190             LOG.warn("Failed to add Route: {} in vpn: {}",
1191                     vrfEntry.getDestPrefix(), rd);
1192             return;
1193         }
1194         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1195             programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
1196         } else {
1197             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1198             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1199                     vpnName, usedRds, vrfEntry.getDestPrefix());
1200             if (!vpnExtraRoutes.isEmpty()) {
1201                 List<InstructionInfo> instructions = new ArrayList<>();
1202                 long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1203                         null, vpnExtraRoutes);
1204                 if (groupId == FibConstants.INVALID_GROUP_ID) {
1205                     LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1206                             vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1207                     return;
1208                 }
1209                 List<ActionInfo> actionInfos =
1210                         Collections.singletonList(new ActionGroup(groupId));
1211                 instructions.add(new InstructionApplyActions(actionInfos));
1212                 makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1213             } else {
1214                 programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
1215             }
1216         }
1217         if (!wrTxPresent) {
1218             tx.submit();
1219         }
1220         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1221     }
1222
1223     private void programRemoteFib(final BigInteger remoteDpnId, final long vpnId,
1224             final VrfEntry vrfEntry, WriteTransaction tx, String rd, List<AdjacencyResult> adjacencyResults) {
1225         List<InstructionInfo> instructions = new ArrayList<>();
1226         for (AdjacencyResult adjacencyResult : adjacencyResults) {
1227             List<ActionInfo> actionInfos = new ArrayList<>();
1228             String egressInterface = adjacencyResult.getInterfaceName();
1229             if (FibUtil.isTunnelInterface(adjacencyResult)) {
1230                 addTunnelInterfaceActions(adjacencyResult, vpnId, vrfEntry, actionInfos);
1231             } else {
1232                 addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1233             }
1234             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(egressInterface,
1235                     actionInfos.size());
1236             if (egressActions.isEmpty()) {
1237                 LOG.error(
1238                         "Failed to retrieve egress action for prefix {} route-paths {} interface {}. "
1239                                 + "Aborting remote FIB entry creation.",
1240                                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
1241                 return;
1242             }
1243             actionInfos.addAll(egressActions);
1244             instructions.add(new InstructionApplyActions(actionInfos));
1245         }
1246         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1247     }
1248
1249     private void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, List<ActionInfo> actionInfos) {
1250         if (vrfEntry.getMac() != null) {
1251             actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
1252                 new MacAddress(vrfEntry.getMac())));
1253             return;
1254         }
1255
1256         String ipPrefix = vrfEntry.getDestPrefix();
1257         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, ipPrefix);
1258         if (prefixInfo == null) {
1259             LOG.debug("No prefix info found for prefix {}", ipPrefix);
1260             return;
1261         }
1262
1263         String ifName = prefixInfo.getVpnInterfaceName();
1264         if (ifName == null) {
1265             LOG.warn("Failed to get VPN interface for prefix {}", ipPrefix);
1266             return;
1267         }
1268
1269         String macAddress = FibUtil.getMacAddressFromPrefix(dataBroker, ifName, ipPrefix);
1270         if (macAddress == null) {
1271             LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix);
1272             return;
1273         }
1274
1275         actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(), new MacAddress(macAddress)));
1276     }
1277
1278     private void addTunnelInterfaceActions(AdjacencyResult adjacencyResult, long vpnId, VrfEntry vrfEntry,
1279                                            List<ActionInfo> actionInfos) {
1280         Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper.getTunnelType(interfaceManager,
1281                 adjacencyResult.getInterfaceName());
1282         // TODO - For now have added routePath into adjacencyResult so that we know for which
1283         // routePath this result is built for. If this is not possible construct a map which does
1284         // the same.
1285         String nextHopIp = adjacencyResult.getNextHopIp();
1286         java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
1287         if (!optionalLabel.isPresent()) {
1288             LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
1289             return;
1290         }
1291         long label = optionalLabel.get();
1292         if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
1293             LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1294             actionInfos.add(new ActionPushMpls());
1295             actionInfos.add(new ActionSetFieldMplsLabel(label));
1296             actionInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
1297         } else {
1298             BigInteger tunnelId;
1299             // FIXME vxlan vni bit set is not working properly with OVS.need to
1300             // revisit
1301             if (tunnelType.equals(TunnelTypeVxlan.class)) {
1302                 Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1303                 // Internet VPN VNI will be used as tun_id for NAT use-cases
1304                 if (prefixInfo.isNatPrefix() && vrfEntry.getL3vni() != null && vrfEntry.getL3vni() != 0) {
1305                     tunnelId = BigInteger.valueOf(vrfEntry.getL3vni());
1306                 } else {
1307                     tunnelId = BigInteger.valueOf(label);
1308                 }
1309             } else {
1310                 tunnelId = BigInteger.valueOf(label);
1311             }
1312
1313             LOG.debug("adding set tunnel id action for label {}", label);
1314             actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
1315             addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1316         }
1317     }
1318
1319     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1320         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1321         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1322         if (dpnInVpn.isPresent()) {
1323             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1324             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1325
1326             if (vpnInterfaces.remove(currVpnInterface)) {
1327                 if (vpnInterfaces.isEmpty()) {
1328                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1329                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1330                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
1331                 } else {
1332                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1333                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1334                         VpnInterfaces.class,
1335                         new VpnInterfacesKey(intfName)));
1336                 }
1337             }
1338         }
1339     }
1340
1341     void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1342     /* Get interface info from prefix to interface mapping;
1343         Use the interface info to get the corresponding vpn interface op DS entry,
1344         remove the adjacency corresponding to this fib entry.
1345         If adjacency removed is the last adjacency, clean up the following:
1346          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1347          - prefix to interface entry
1348          - vpn interface op DS
1349      */
1350         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1351         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1352         Routes extraRoute = null;
1353         if (prefixInfo == null) {
1354             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1355             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1356             extraRoute = getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1357             if (extraRoute != null) {
1358                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1359                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1360                     if (nextHopIp != null) {
1361                         prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp
1362                                 + NwConstants.IPV4PREFIX);
1363                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1364                     }
1365                 }
1366             }
1367             if (prefixInfo == null) {
1368                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1369                 if (optionalLabel.isPresent()) {
1370                     Long label = optionalLabel.get();
1371                     List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1372                     LabelRouteInfo lri = getLabelRouteInfo(label);
1373                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1374                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1375                         prefixBuilder.setDpnId(lri.getDpnId());
1376                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1377                         prefixBuilder.setIpAddress(lri.getPrefix());
1378                         prefixInfo = prefixBuilder.build();
1379                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1380                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1381                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1382                     }
1383                 }
1384             }
1385         } else {
1386             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1387         }
1388     }
1389
1390     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1391                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1392
1393         if (prefixInfo == null) {
1394             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1395             return; //Don't have any info for this prefix (shouldn't happen); need to return
1396         }
1397
1398         if (Boolean.TRUE.equals(prefixInfo.isNatPrefix())) {
1399             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1400                     vrfEntry.getDestPrefix(), vpnId, rd);
1401             return;
1402         }
1403
1404         String ifName = prefixInfo.getVpnInterfaceName();
1405         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1406         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1407             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1408     }
1409
1410     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1411         Prefixes prefixInfo;
1412         Long vpnId;
1413         String rd;
1414         VrfEntry vrfEntry;
1415         Routes extraRoute;
1416
1417         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1418                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1419             this.prefixInfo = prefixInfo;
1420             this.vpnId = vpnId;
1421             this.rd = rd;
1422             this.vrfEntry = vrfEntry;
1423             this.extraRoute = extraRoute;
1424         }
1425
1426         @Override
1427         public List<ListenableFuture<Void>> call() throws Exception {
1428             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1429             // to call the respective helpers.
1430             WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1431
1432             //First Cleanup LabelRouteInfo
1433             //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1434             if (vrfEntry.getEncapType().equals(VrfEntry.EncapType.Mplsgre)) {
1435                 FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1436                     List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1437                     synchronized (label.toString().intern()) {
1438                         LabelRouteInfo lri = getLabelRouteInfo(label);
1439                         if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1440                                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1441                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1442                                     FibUtil.getVpnInstanceOpData(dataBroker, rd);
1443                             String vpnInstanceName = "";
1444                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
1445                                 vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1446                             }
1447                             boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, writeOperTxn);
1448                             if (lriRemoved) {
1449                                 String parentRd = lri.getParentVpnRd();
1450                                 FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1451                                         FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1452                             }
1453                         } else {
1454                             FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1455                                     FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1456                         }
1457                     }
1458                 });
1459             }
1460             String ifName = prefixInfo.getVpnInterfaceName();
1461             Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1462                 FibUtil.getVpnInterfaceIdentifier(ifName));
1463             if (optvpnInterface.isPresent()) {
1464                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1465                 if (vpnId != associatedVpnId) {
1466                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1467                         vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1468                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1469                     return null;
1470                 } else {
1471                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1472                         vrfEntry.getDestPrefix(), associatedVpnId);
1473                 }
1474             }
1475             if (extraRoute != null) {
1476                 Optional<String> optVpnName = FibUtil.getVpnNameFromRd(dataBroker, rd);
1477                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1478                 //Only one used Rd present in case of removal event
1479                 String usedRd = usedRds.get(0);
1480                 if (optVpnName.isPresent()) {
1481                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL,
1482                             getVpnToExtrarouteIdentifier(optVpnName.get(), usedRd, vrfEntry.getDestPrefix()));
1483                     writeOperTxn.delete(LogicalDatastoreType.CONFIGURATION,
1484                             VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix()));
1485                 }
1486             }
1487             Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1488                 FibUtil.getAdjListPath(ifName));
1489             int numAdj = 0;
1490             if (optAdjacencies.isPresent()) {
1491                 numAdj = optAdjacencies.get().getAdjacency().size();
1492             }
1493             //remove adjacency corr to prefix
1494             if (numAdj > 1) {
1495                 LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId,
1496                         vrfEntry.getDestPrefix());
1497                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL,
1498                         FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1499             } else {
1500                 //this is last adjacency (or) no more adjacency left for this vpn interface, so
1501                 //clean up the vpn interface from DpnToVpn list
1502                 LOG.info("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1503                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, FibUtil.getVpnInterfaceIdentifier(ifName));
1504             }
1505             List<ListenableFuture<Void>> futures = new ArrayList<>();
1506             futures.add(writeOperTxn.submit());
1507             return futures;
1508         }
1509     }
1510
1511     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1512         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1513         final String rd = vrfTableKey.getRouteDistinguisher();
1514         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1515         if (vpnInstance == null) {
1516             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1517             return;
1518         }
1519         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1520         long elanTag = 0L;
1521         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1522         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1523         List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1524         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId());
1525         if (subnetRoute != null) {
1526             elanTag = subnetRoute.getElantag();
1527             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1528                 rd, vrfEntry.getDestPrefix(), elanTag);
1529             if (vpnToDpnList != null) {
1530                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1531                 dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(),
1532                     () -> {
1533                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1534
1535                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1536
1537                             makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1538                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1539                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1540                                 optionalLabel.ifPresent(label -> {
1541                                     makeLFibTableEntry(curDpn.getDpnId(), label, null,
1542                                             DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1543                                 });
1544                             }
1545                         }
1546                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1547                         futures.add(tx.submit());
1548                         return futures;
1549                     });
1550             }
1551             optionalLabel.ifPresent(label -> {
1552                 synchronized (label.toString().intern()) {
1553                     LabelRouteInfo lri = getLabelRouteInfo(label);
1554                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1555                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1556                                 FibUtil.getVpnInstanceOpData(dataBroker, rd);
1557                         String vpnInstanceName = "";
1558                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1559                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1560                         }
1561                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1562                         if (lriRemoved) {
1563                             String parentRd = lri.getParentVpnRd();
1564                             FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1565                                     FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1566                             LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as "
1567                                     + "labelRouteInfo cleared", label, rd,
1568                                     vrfEntry.getDestPrefix());
1569                         }
1570                     } else {
1571                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1572                                 FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1573                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1574                                 label, rd, vrfEntry.getDestPrefix());
1575                     }
1576                 }
1577             });
1578             return;
1579         }
1580         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnInstance.getVpnId(), NwConstants.DEL_FLOW)) {
1581             return;
1582         }
1583
1584         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1585             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1586         if (vpnToDpnList != null) {
1587             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1588                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1589             String usedRd = null;
1590             Optional<Routes> extraRouteOptional;
1591             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1592             if (usedRds != null && !usedRds.isEmpty()) {
1593                 if (usedRds.size() > 1) {
1594                     LOG.error("The extra route prefix is still present in some DPNs");
1595                     return ;
1596                 } else {
1597                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1598                     //is not present in any other DPN
1599                     extraRouteOptional = VpnExtraRouteHelper
1600                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1601                 }
1602             } else {
1603                 extraRouteOptional = Optional.absent();
1604             }
1605             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1606             dataStoreCoordinator.enqueueJob("FIB-" + usedRd + "-" + vrfEntry.getDestPrefix(),
1607                 () -> {
1608                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1609
1610                     if (localDpnIdList.size() <= 0) {
1611                         for (VpnToDpnList curDpn : vpnToDpnList) {
1612                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1613                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1614                                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1615                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1616                                 }
1617                             } else {
1618                                 deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1619                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1620                             }
1621                         }
1622                     } else {
1623                         for (BigInteger localDpnId : localDpnIdList) {
1624                             for (VpnToDpnList curDpn : vpnToDpnList) {
1625                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1626                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1627                                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1628                                             deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1629                                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1630                                         }
1631                                     } else {
1632                                         deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1633                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1634                                     }
1635                                 }
1636                             }
1637                         }
1638                     }
1639                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1640                     futures.add(tx.submit());
1641                     return futures;
1642                 });
1643         }
1644
1645         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1646         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1647         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1648
1649         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1650         // of the interVpnLink.
1651         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(this.dataBroker, rd);
1652         if (optVpnUuid.isPresent()) {
1653             String vpnUuid = optVpnUuid.get();
1654             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1655                 Optional<InterVpnLinkDataComposite> optInterVpnLink =
1656                         InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1657                 if (optInterVpnLink.isPresent()) {
1658                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1659                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1660                         // This is route that points to the other endpoint of an InterVpnLink
1661                         // In that case, we should look for the FIB table pointing to
1662                         // LPortDispatcher table and remove it.
1663                         removeInterVPNLinkRouteFlows(interVpnLink.getInterVpnLinkName(),
1664                                 interVpnLink.isFirstEndpointVpnName(rd),
1665                                 vrfEntry);
1666                     }
1667                 }
1668             });
1669         }
1670
1671     }
1672
1673     /*
1674       Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1675       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1676       provided by ResourceBatchingManager
1677      */
1678     private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
1679                                   final VrfEntry vrfEntry) {
1680         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1681         String rd = vrfTableKey.getRouteDistinguisher();
1682         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1683         if (vpnInstance == null) {
1684             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1685             return;
1686         }
1687         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId());
1688         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1689         if (vpnToDpnList != null) {
1690             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1691                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1692             Optional<Routes> extraRouteOptional;
1693             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1694             if (usedRds != null && !usedRds.isEmpty()) {
1695                 if (usedRds.size() > 1) {
1696                     LOG.error("The extra route prefix is still present in some DPNs");
1697                     return ;
1698                 } else {
1699                     extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1700                             usedRds.get(0), vrfEntry.getDestPrefix());
1701                 }
1702             } else {
1703                 extraRouteOptional = Optional.absent();
1704             }
1705             for (VpnToDpnList curDpn : vpnToDpnList) {
1706                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1707                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey,
1708                             vrfEntry, extraRouteOptional, writeTx);
1709                 }
1710             }
1711         }
1712     }
1713
1714     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1715             final long vpnId, final VrfTablesKey vrfTableKey,
1716             final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional, WriteTransaction tx) {
1717
1718         Boolean wrTxPresent = true;
1719         if (tx == null) {
1720             wrTxPresent = false;
1721             tx = dataBroker.newWriteOnlyTransaction();
1722         }
1723
1724         LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
1725                 vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
1726         String rd = vrfTableKey.getRouteDistinguisher();
1727
1728         if (localDpnId != null && localDpnId != BigInteger.ZERO) {
1729             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1730             if (extraRouteOptional.isPresent()) {
1731                 nextHopManager.setupLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix(), null , false);
1732             }
1733             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1734             return;
1735         }
1736
1737         // below two reads are kept as is, until best way is found to identify dpnID
1738         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1739         if (extraRouteOptional.isPresent()) {
1740             nextHopManager.setupLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix(), null , false);
1741         } else {
1742             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1743         }
1744         if (!wrTxPresent) {
1745             tx.submit();
1746         }
1747     }
1748
1749     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1750                                            VrfEntry vrfEntry, String rd, WriteTransaction tx) {
1751         boolean isRemoteRoute = true;
1752         if (localNextHopInfo != null) {
1753             isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId());
1754         }
1755         if (isRemoteRoute) {
1756             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1757             return true;
1758         } else {
1759             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}",
1760                 rd, vrfEntry.getDestPrefix(), remoteDpnId);
1761             return false;
1762         }
1763     }
1764
1765     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry,
1766             String rd, WriteTransaction tx) {
1767         // When the tunnel is removed the fib entries should be reprogrammed/deleted depending on
1768         // the adjacencyResults.
1769         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1770             List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1771             if (!adjacencyResults.isEmpty()) {
1772                 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
1773                 return;
1774             }
1775         }
1776         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1777         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1778     }
1779
1780     private long get(byte[] rawIpAddress) {
1781         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1782             + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1783     }
1784
1785     void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1786                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1787         Boolean wrTxPresent = true;
1788         if (tx == null) {
1789             wrTxPresent = false;
1790             tx = dataBroker.newWriteOnlyTransaction();
1791         }
1792
1793         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1794         String[] values = vrfEntry.getDestPrefix().split("/");
1795         String ipAddress = values[0];
1796         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1797         if (addOrRemove == NwConstants.ADD_FLOW) {
1798             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1799         } else {
1800             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1801         }
1802         InetAddress destPrefix;
1803         try {
1804             destPrefix = InetAddress.getByName(ipAddress);
1805         } catch (UnknownHostException e) {
1806             LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1807             return;
1808         }
1809
1810         List<MatchInfo> matches = new ArrayList<>();
1811
1812         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
1813
1814         if (destPrefix instanceof Inet4Address) {
1815             matches.add(MatchEthernetType.IPV4);
1816             if (prefixLength != 0) {
1817                 matches.add(new MatchIpv4Destination(destPrefix.getHostAddress(), Integer.toString(prefixLength)));
1818             }
1819         } else {
1820             matches.add(MatchEthernetType.IPV6);
1821             if (prefixLength != 0) {
1822                 matches.add(new MatchIpv6Destination(destPrefix.getHostAddress() + "/" + prefixLength));
1823             }
1824         }
1825
1826         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1827         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1828         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority,
1829             flowRef, 0, 0,
1830             COOKIE_VM_FIB_TABLE, matches, instructions);
1831         Flow flow = flowEntity.getFlowBuilder().build();
1832         String flowId = flowEntity.getFlowId();
1833         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1834         Node nodeDpn = buildDpnNode(dpId);
1835
1836         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1837             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1838             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1839
1840         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1841             SubTransaction subTransaction = new SubTransactionImpl();
1842             if (addOrRemove == NwConstants.ADD_FLOW) {
1843                 subTransaction.setInstanceIdentifier(flowInstanceId);
1844                 subTransaction.setInstance(flow);
1845                 subTransaction.setAction(SubTransaction.CREATE);
1846             } else {
1847                 subTransaction.setInstanceIdentifier(flowInstanceId);
1848                 subTransaction.setAction(SubTransaction.DELETE);
1849             }
1850             transactionObjects.add(subTransaction);
1851         }
1852
1853         if (addOrRemove == NwConstants.ADD_FLOW) {
1854             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
1855         } else {
1856             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1857         }
1858
1859         if (!wrTxPresent) {
1860             tx.submit();
1861         }
1862     }
1863
1864     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1865     Node buildDpnNode(BigInteger dpnId) {
1866         NodeId nodeId = new NodeId("openflow:" + dpnId);
1867         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1868
1869         return nodeDpn;
1870     }
1871
1872     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1873                                     int addOrRemove, WriteTransaction tx) {
1874         Boolean wrTxPresent = true;
1875         if (tx == null) {
1876             wrTxPresent = false;
1877             tx = dataBroker.newWriteOnlyTransaction();
1878         }
1879
1880         List<MatchInfo> matches = new ArrayList<>();
1881         matches.add(MatchEthernetType.MPLS_UNICAST);
1882         matches.add(new MatchMplsLabel(label));
1883
1884         // Install the flow entry in L3_LFIB_TABLE
1885         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1886
1887         FlowEntity flowEntity;
1888         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1889             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1890         Flow flow = flowEntity.getFlowBuilder().build();
1891         String flowId = flowEntity.getFlowId();
1892         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1893         Node nodeDpn = buildDpnNode(dpId);
1894         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1895             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1896             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1897
1898         if (addOrRemove == NwConstants.ADD_FLOW) {
1899             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
1900         } else {
1901             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1902         }
1903         if (!wrTxPresent) {
1904             tx.submit();
1905         }
1906
1907         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1908             dpId, label, instructions, flowKey, (NwConstants.ADD_FLOW == addOrRemove) ? "ADDED" : "REMOVED");
1909     }
1910
1911     void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress,
1912                                       final String ipPrefixAddress) {
1913         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}", dpId, vpnId, ipAddress);
1914         try {
1915             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress);
1916         } catch (NullPointerException e) {
1917             LOG.trace("", e);
1918         }
1919     }
1920
1921     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1922                                     final FutureCallback<List<Void>> callback) {
1923         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1924         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1925         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1926         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1927         if (!vrfTable.isPresent()) {
1928             LOG.warn("VRF Table not yet available for RD {}", rd);
1929             if (callback != null) {
1930                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1931                 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1932                 Futures.addCallback(listenableFuture, callback);
1933             }
1934             return;
1935         }
1936         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1937         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1938             () -> {
1939                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1940                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1941                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1942                     for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1943                         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1944                         if (subnetRoute != null) {
1945                             long elanTag = subnetRoute.getElantag();
1946                             installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1947                             continue;
1948                         }
1949                         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1950                         if (routerInt != null) {
1951                             LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1952                                 rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1953                             installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
1954                                 routerInt.getIpAddress(),
1955                                 new MacAddress(routerInt.getMacAddress()), NwConstants.ADD_FLOW);
1956                             continue;
1957                         }
1958                         //Handle local flow creation for imports
1959                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1960                             java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1961                             if (optionalLabel.isPresent()) {
1962                                 List<String> nextHopList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1963                                 LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1964                                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1965                                     if (lri.getDpnId().equals(dpnId)) {
1966                                         createLocalFibEntry(vpnId, rd, vrfEntry);
1967                                         continue;
1968                                     }
1969                                 }
1970                             }
1971                         }
1972
1973                         boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1974                                 vrfEntry, dpnId);
1975                         if (shouldCreateRemoteFibEntry) {
1976                             LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}",
1977                                     vrfEntry, dpnId);
1978                             createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1979                         }
1980                     }
1981                     //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1982                     futures.add(tx.submit());
1983                 }
1984                 if (callback != null) {
1985                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1986                     Futures.addCallback(listenableFuture, callback);
1987                 }
1988                 return futures;
1989             });
1990     }
1991
1992     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1993                                             final String localNextHopIp, final String remoteNextHopIp) {
1994         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1995             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1996         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1997         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1998         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1999         if (vrfTable.isPresent()) {
2000             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2001             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2002                 () -> {
2003                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2004                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2005                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
2006                         vrfTable.get().getVrfEntry().stream()
2007                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
2008                             .forEach(
2009                                 getConsumerForCreatingRemoteFib(dpnId, vpnId,
2010                                         rd, remoteNextHopIp, vrfTable,
2011                                         writeCfgTxn));
2012                         futures.add(writeCfgTxn.submit());
2013                     }
2014                     return futures;
2015                 });
2016         }
2017     }
2018
2019     public void populateInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
2020                                             final String localNextHopIp, final String remoteNextHopIp) {
2021         LOG.trace("populateInternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
2022             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2023         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2024         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2025         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2026         if (vrfTable.isPresent()) {
2027             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2028             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2029                 () -> {
2030                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2031                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2032                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
2033                         // Handle Vpn Interface driven Routes only (i.e., STATIC and LOCAL)
2034                         vrfTable.get().getVrfEntry().stream()
2035                             .filter(vrfEntry -> {
2036                                 /* Ignore SubnetRoute entry */
2037                                 return (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
2038                                         vrfEntry.getOrigin())));
2039                             })
2040                             .forEach(getConsumerForCreatingRemoteFib(dpnId, vpnId,
2041                                        rd, remoteNextHopIp, vrfTable,
2042                                        writeCfgTxn));
2043                         futures.add(writeCfgTxn.submit());
2044                     }
2045                     return futures;
2046                 });
2047         }
2048     }
2049
2050     public void manageRemoteRouteOnDPN(final boolean action,
2051                                        final BigInteger localDpnId,
2052                                        final long vpnId,
2053                                        final String rd,
2054                                        final String destPrefix,
2055                                        final String destTepIp,
2056                                        final long label) {
2057         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2058
2059         if (vpnInstance == null) {
2060             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
2061             return;
2062         }
2063         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2064         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + localDpnId.toString(),
2065             () -> {
2066                 List<ListenableFuture<Void>> futures = new ArrayList<>();
2067                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
2068                     WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
2069                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
2070                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
2071                     if (vrfEntry == null) {
2072                         return futures;
2073                     }
2074                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
2075                         action, localDpnId, vpnId, rd, destPrefix);
2076                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
2077                     VrfEntry modVrfEntry;
2078                     if (routePathList == null || (routePathList.isEmpty())) {
2079                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
2080                                 Collections.singletonList(destTepIp),
2081                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
2082                     } else {
2083                         modVrfEntry = vrfEntry;
2084                     }
2085
2086                     if (action == true) {
2087                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
2088                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
2089                     } else {
2090                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
2091                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
2092                                 vrfEntry.getDestPrefix());
2093                         if (usedRds.size() > 1) {
2094                             LOG.debug("The extra route prefix is still present in some DPNs");
2095                             return futures;
2096                         }
2097                         //Is this fib route an extra route? If yes, get the nexthop which would be
2098                         //an adjacency in the vpn
2099                         Optional<Routes> extraRouteOptional = Optional.absent();
2100                         if (usedRds.size() != 0) {
2101                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
2102                                     FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId()),
2103                                     usedRds.get(0), vrfEntry.getDestPrefix());
2104                         }
2105                         deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
2106                                 extraRouteOptional, writeTransaction);
2107                     }
2108                     futures.add(writeTransaction.submit());
2109                 }
2110                 return futures;
2111             });
2112     }
2113
2114     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
2115                                  final FutureCallback<List<Void>> callback) {
2116         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
2117         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2118         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2119         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2120         if (vrfTable.isPresent()) {
2121             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2122             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2123                 () -> {
2124                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2125                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2126                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
2127                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
2128                                 /* Handle subnet routes here */
2129                             SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
2130                             if (subnetRoute != null) {
2131                                 LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn",
2132                                         vrfEntry.getDestPrefix(),
2133                                         dpnId, rd);
2134                                 makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
2135                                 java.util.Optional.ofNullable(vrfEntry.getRoutePaths()).ifPresent(routePaths -> {
2136                                     routePaths.stream().forEach(routePath -> {
2137                                         makeLFibTableEntry(dpnId, routePath.getLabel(), null,
2138                                                 DEFAULT_FIB_FLOW_PRIORITY,
2139                                                 NwConstants.DEL_FLOW, tx);
2140                                         LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} "
2141                                                 + "for rd {} prefix {}",
2142                                                 routePath.getLabel(), rd,
2143                                                 vrfEntry.getDestPrefix());
2144                                     });
2145                                 });
2146                                 continue;
2147                             }
2148                             // ping responder for router interfaces
2149                             RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
2150                             if (routerInt != null) {
2151                                 LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
2152                                     rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
2153                                 installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
2154                                     routerInt.getIpAddress(),
2155                                     new MacAddress(routerInt.getMacAddress()), NwConstants.DEL_FLOW);
2156                                 continue;
2157                             }
2158                             // Passing null as we don't know the dpn
2159                             // to which prefix is attached at this point
2160                             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
2161                                     vrfEntry.getDestPrefix());
2162                             String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId());
2163                             Optional<Routes> extraRouteOptional;
2164                             //Is this fib route an extra route? If yes, get the nexthop which would be
2165                             //an adjacency in the vpn
2166                             if (usedRds != null && !usedRds.isEmpty()) {
2167                                 if (usedRds.size() > 1) {
2168                                     LOG.error("The extra route prefix is still present in some DPNs");
2169                                     return futures;
2170                                 } else {
2171                                     extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
2172                                             usedRds.get(0), vrfEntry.getDestPrefix());
2173
2174                                 }
2175                             } else {
2176                                 extraRouteOptional = Optional.absent();
2177                             }
2178                             deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry,
2179                                     extraRouteOptional, tx);
2180                         }
2181                         futures.add(tx.submit());
2182                         if (callback != null) {
2183                             ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
2184                             Futures.addCallback(listenableFuture, callback);
2185                         }
2186                     }
2187                     return futures;
2188                 });
2189         }
2190     }
2191
2192     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
2193                                            final String localNextHopIp, final String remoteNextHopIp) {
2194         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
2195                 + " localNexthopIp {} , remoteNexhtHopIp {}",
2196             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2197         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2198         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2199         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2200         if (vrfTable.isPresent()) {
2201             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2202             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2203                 () -> {
2204                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2205                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2206                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
2207                         vrfTable.get().getVrfEntry().stream()
2208                             .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
2209                             .forEach(getConsumerForDeletingRemoteFib(dpnId, vpnId, rd,
2210                                 remoteNextHopIp, vrfTable, writeCfgTxn));
2211                         futures.add(writeCfgTxn.submit());
2212                     }
2213                     return futures;
2214                 });
2215         }
2216     }
2217
2218     public void cleanUpInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
2219                                            final String localNextHopIp, final String remoteNextHopIp) {
2220         LOG.trace("cleanUpInternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
2221                 + " localNexthopIp {} , remoteNexhtHopIp {}",
2222             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2223         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2224         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2225         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2226         if (vrfTable.isPresent()) {
2227             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2228             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2229                 () -> {
2230                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2231                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2232                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
2233                         vrfTable.get().getVrfEntry().stream()
2234                             .filter(vrfEntry -> {
2235                                 /* Ignore SubnetRoute entry */
2236                                 return (FibHelper.isControllerManagedVpnInterfaceRoute(
2237                                         RouteOrigin.value(vrfEntry.getOrigin())));
2238                             })
2239                             .forEach(getConsumerForDeletingRemoteFib(dpnId, vpnId,
2240                                 rd, remoteNextHopIp, vrfTable, writeCfgTxn));
2241                         futures.add(writeCfgTxn.submit());
2242                     }
2243                     return futures;
2244                 });
2245         }
2246     }
2247
2248     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
2249         InstanceIdentifierBuilder<VrfTables> idBuilder =
2250             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
2251         InstanceIdentifier<VrfTables> id = idBuilder.build();
2252         return id;
2253     }
2254
2255     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
2256         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + label
2257                 + NwConstants.FLOWID_SEPARATOR + priority;
2258     }
2259
2260     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
2261         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + rd
2262                 + NwConstants.FLOWID_SEPARATOR + priority + NwConstants.FLOWID_SEPARATOR + destPrefix.getHostAddress();
2263     }
2264
2265     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
2266         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
2267                 .FLOWID_SEPARATOR + nextHop;
2268     }
2269
2270     protected List<AdjacencyResult> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
2271                                                      final VrfEntry vrfEntry, String rd) {
2272         List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
2273         FibHelper.sortIpAddress(routePaths);
2274         List<AdjacencyResult> adjacencyList = new ArrayList<>();
2275         List<String> prefixIpList = new ArrayList<>();
2276         LOG.trace("resolveAdjacency called with remotedDpnId {}, vpnId{}, VrfEntry {}",
2277             remoteDpnId, vpnId, vrfEntry);
2278         try {
2279             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2280                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
2281                 List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
2282                         FibUtil.getVpnNameFromId(dataBroker, vpnId), usedRds, vrfEntry.getDestPrefix());
2283                 if (vpnExtraRoutes.isEmpty()) {
2284                     prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
2285                 } else {
2286                     List<String> prefixIpListLocal = new ArrayList<>();
2287                     vpnExtraRoutes.stream().forEach(route -> {
2288                         route.getNexthopIpList().stream().forEach(extraRouteIp -> {
2289                             prefixIpListLocal.add(extraRouteIp + NwConstants.IPV4PREFIX);
2290                         });
2291                     });
2292                     prefixIpList = prefixIpListLocal;
2293                 }
2294             } else {
2295                 prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
2296             }
2297
2298             for (String prefixIp : prefixIpList) {
2299                 if (routePaths == null || routePaths.isEmpty()) {
2300                     LOG.trace("Processing Destination IP {} without NextHop IP", prefixIp);
2301                     AdjacencyResult adjacencyResult = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
2302                           prefixIp, null);
2303                     addAdjacencyResultToList(adjacencyList, adjacencyResult);
2304                     continue;
2305                 }
2306                 adjacencyList.addAll(routePaths.stream()
2307                         .map(routePath -> {
2308                             LOG.debug("NextHop IP for destination {} is {}", prefixIp,
2309                                     routePath.getNexthopAddress());
2310                             return nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
2311                                     prefixIp, routePath.getNexthopAddress());
2312                         })
2313                         .filter(adjacencyResult -> adjacencyResult != null && !adjacencyList.contains(adjacencyResult))
2314                         .distinct()
2315                         .collect(toList()));
2316             }
2317         } catch (NullPointerException e) {
2318             LOG.trace("", e);
2319         }
2320         return adjacencyList;
2321     }
2322
2323     private void addAdjacencyResultToList(List<AdjacencyResult> adjacencyList, AdjacencyResult adjacencyResult) {
2324         if (adjacencyResult != null && !adjacencyList.contains(adjacencyResult)) {
2325             adjacencyList.add(adjacencyResult);
2326         }
2327     }
2328
2329     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
2330         InstanceIdentifier<VpnInstanceOpDataEntry> id =
2331             InstanceIdentifier.create(VpnInstanceOpData.class)
2332                 .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
2333         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
2334             FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
2335         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
2336     }
2337
2338     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2339         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
2340                 + tableMiss + FLOWID_PREFIX;
2341     }
2342
2343     /*
2344      * Install flow entry in protocol table to forward mpls
2345      * coming through gre tunnel to LFIB table.
2346      */
2347     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
2348         final BigInteger cookieProtocolTable = new BigInteger("1070000", 16);
2349         // Instruction to goto L3 InterfaceTable
2350         List<InstructionInfo> instructions = new ArrayList<>();
2351         instructions.add(new InstructionGotoTable(NwConstants.L3_LFIB_TABLE));
2352         List<MatchInfo> matches = new ArrayList<>();
2353         matches.add(MatchEthernetType.MPLS_UNICAST);
2354         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2355             getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2356                 NwConstants.L3_LFIB_TABLE),
2357             DEFAULT_FIB_FLOW_PRIORITY,
2358             "Protocol Table For LFIB",
2359             0, 0,
2360             cookieProtocolTable,
2361             matches, instructions);
2362
2363         if (addOrRemove == NwConstants.ADD_FLOW) {
2364             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2365             mdsalManager.installFlow(flowEntityToLfib);
2366         } else {
2367             mdsalManager.removeFlow(flowEntityToLfib);
2368         }
2369     }
2370
2371     public List<String> printFibEntries() {
2372         List<String> result = new ArrayList<>();
2373         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2374         result.add("-------------------------------------------------------------------");
2375         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2376         Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2377         if (fibEntries.isPresent()) {
2378             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2379             for (VrfTables vrfTable : vrfTables) {
2380                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2381                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
2382                     for (RoutePaths routePath : routePaths) {
2383                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2384                             vrfTable.getRouteDistinguisher(),
2385                             vrfEntry.getDestPrefix(), routePath.getNexthopAddress(),
2386                             routePath.getLabel(), vrfEntry.getOrigin()));
2387                     }
2388                     if (routePaths.isEmpty()) {
2389                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s",
2390                             vrfTable.getRouteDistinguisher(),
2391                             vrfEntry.getDestPrefix(), "local", vrfEntry.getOrigin()));
2392                     }
2393                 }
2394             }
2395         }
2396         return result;
2397     }
2398
2399
2400     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2401         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2402             .child(VrfTables.class, new VrfTablesKey(rd))
2403             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2404         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2405         if (vrfEntry.isPresent()) {
2406             return vrfEntry.get();
2407         }
2408         return null;
2409     }
2410
2411     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2412         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2413             .child(VrfTables.class, new VrfTablesKey(rd))
2414             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2415         return vrfEntryId;
2416     }
2417
2418     protected Boolean isIpv4Address(String ipAddress) {
2419         try {
2420             InetAddress address = InetAddress.getByName(ipAddress);
2421             if (address instanceof Inet4Address) {
2422                 return true;
2423             }
2424         } catch (UnknownHostException e) {
2425             LOG.warn("Invalid ip address {}", ipAddress, e);
2426             return false;
2427         }
2428         return false;
2429     }
2430
2431     protected Boolean installRouterFibEntries(final VrfEntry vrfEntry, final Collection<VpnToDpnList> vpnToDpnList,
2432                                               long vpnId, int addOrRemove) {
2433         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
2434         if (routerInt == null) {
2435             return false;
2436         }
2437         if (vpnToDpnList != null) {
2438             String routerId = routerInt.getUuid();
2439             String macAddress = routerInt.getMacAddress();
2440             String ipValue = routerInt.getIpAddress();
2441             LOG.trace("createFibEntries - Router augmented vrfentry found for for router uuid:{}, ip:{}, mac:{}",
2442                     routerId, ipValue, macAddress);
2443             for (VpnToDpnList vpnDpn : vpnToDpnList) {
2444                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
2445                     installRouterFibEntry(vrfEntry, vpnDpn.getDpnId(), vpnId, routerId, ipValue,
2446                             new MacAddress(macAddress), addOrRemove);
2447                 }
2448             }
2449         }
2450         return true;
2451     }
2452
2453     public void installRouterFibEntry(final VrfEntry vrfEntry, BigInteger dpnId, long vpnId, String routerUuid,
2454                                       String routerInternalIp, MacAddress routerMac, int addOrRemove) {
2455         String[] subSplit = routerInternalIp.split("/");
2456         if (!isIpv4Address(subSplit[0])) {
2457             // Ping responder using OpenFlow rules is only supported for IPv4, hence skipping.
2458             return;
2459         }
2460
2461         java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2462         if (!optionalLabel.isPresent()) {
2463             LOG.warn("Routes paths not present. Exiting installRouterFibEntry");
2464             return;
2465         }
2466         String addRemoveStr = (addOrRemove == NwConstants.ADD_FLOW) ? "ADD_FLOW" : "DELETE_FLOW";
2467         LOG.trace("{}: bulding Echo Flow entity for dpid:{}, router_ip:{}, vpnId:{}, subSplit:{} ", addRemoveStr,
2468             dpnId, routerInternalIp, vpnId, subSplit[0]);
2469
2470         List<MatchInfo> matches = new ArrayList<>();
2471
2472         matches.add(MatchIpProtocol.ICMP);
2473         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
2474         matches.add(new MatchIcmpv4((short) 8, (short) 0));
2475         matches.add(MatchEthernetType.IPV4);
2476         matches.add(new MatchIpv4Destination(subSplit[0], "32"));
2477
2478         List<ActionInfo> actionsInfos = new ArrayList<>();
2479
2480         // Set Eth Src and Eth Dst
2481         actionsInfos.add(new ActionMoveSourceDestinationEth());
2482         actionsInfos.add(new ActionSetFieldEthernetSource(routerMac));
2483
2484         // Move Ip Src to Ip Dst
2485         actionsInfos.add(new ActionMoveSourceDestinationIp());
2486         actionsInfos.add(new ActionSetSourceIp(subSplit[0], "32"));
2487
2488         // Set the ICMP type to 0 (echo reply)
2489         actionsInfos.add(new ActionSetIcmpType((short) 0));
2490
2491         actionsInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
2492
2493         actionsInfos.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2494
2495         List<InstructionInfo> instructions = new ArrayList<>();
2496
2497         instructions.add(new InstructionApplyActions(actionsInfos));
2498
2499         int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH;
2500         String flowRef = getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, optionalLabel.get(),
2501                 priority);
2502
2503         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef,
2504             0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions);
2505
2506         if (addOrRemove == NwConstants.ADD_FLOW) {
2507             mdsalManager.installFlow(flowEntity);
2508         } else {
2509             mdsalManager.removeFlow(flowEntity);
2510         }
2511     }
2512
2513     public void removeInterVPNLinkRouteFlows(final String interVpnLinkName,
2514                                              final boolean isVpnFirstEndPoint,
2515                                              final VrfEntry vrfEntry) {
2516         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
2517             && vrfEntry.getRoutePaths().size() == 1);
2518         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLinkName);
2519
2520         if (!interVpnLinkState.isPresent()) {
2521             LOG.warn("Could not find State for InterVpnLink {}", interVpnLinkName);
2522             return;
2523         }
2524
2525         List<BigInteger> targetDpns =
2526             isVpnFirstEndPoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
2527                 : interVpnLinkState.get().getSecondEndpointState().getDpId();
2528
2529         java.util.Optional<String> optionalNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
2530         java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2531
2532         // delete from FIB
2533         //
2534         optionalNextHop.ifPresent(nextHop -> {
2535             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2536             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2537             Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef))
2538                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
2539
2540             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}",
2541                     interVpnLinkName, flowRef);
2542
2543             for (BigInteger dpId : targetDpns) {
2544                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2545                         vrfEntry.getDestPrefix(), nextHop,
2546                         dpId, interVpnLinkName);
2547
2548                 mdsalManager.removeFlow(dpId, flow);
2549             }
2550         });
2551
2552         // delete from LFIB
2553         //
2554         optionalLabel.ifPresent(label -> {
2555             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2556
2557             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
2558             for (BigInteger dpId : targetDpns) {
2559                 LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
2560                         vrfEntry.getDestPrefix(), label,
2561                         dpId, interVpnLinkName);
2562                 makeLFibTableEntry(dpId, label, null /* no instructions */,
2563                         LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, tx);
2564             }
2565             tx.submit();
2566         });
2567     }
2568
2569     private Consumer<? super VrfEntry> getConsumerForCreatingRemoteFib(
2570             final BigInteger dpnId, final long vpnId, final String rd,
2571             final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
2572             WriteTransaction writeCfgTxn) {
2573         return vrfEntry -> vrfEntry.getRoutePaths().stream()
2574                 .filter(routes -> !routes.getNexthopAddress().isEmpty()
2575                         && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
2576                 .findFirst()
2577                 .ifPresent(routes -> {
2578                     LOG.trace("creating remote FIB entry for prefix {} rd {} on Dpn {}",
2579                             vrfEntry.getDestPrefix(), rd, dpnId);
2580                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeCfgTxn);
2581                 });
2582     }
2583
2584     private Consumer<? super VrfEntry> getConsumerForDeletingRemoteFib(
2585             final BigInteger dpnId, final long vpnId, final String rd,
2586             final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
2587             WriteTransaction writeCfgTxn) {
2588         return vrfEntry -> vrfEntry.getRoutePaths().stream()
2589                 .filter(routes -> !routes.getNexthopAddress().isEmpty()
2590                         && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
2591                 .findFirst()
2592                 .ifPresent(routes -> {
2593                     LOG.trace(" deleting remote FIB entry {}", vrfEntry);
2594                     deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry,
2595                             Optional.absent(), writeCfgTxn);
2596                 });
2597     }
2598
2599     private boolean isPrefixAndNextHopPresentInLri(String prefix,
2600             List<String> nextHopAddressList, LabelRouteInfo lri) {
2601         return lri != null && lri.getPrefix().equals(prefix)
2602                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
2603     }
2604
2605     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
2606         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2607             return true;
2608         }
2609
2610         Prefixes prefix = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
2611         if (prefix != null) {
2612             BigInteger prefixDpnId = prefix.getDpnId();
2613             if (prefixDpnId == dpnId) {
2614                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2615                         vrfEntry, dpnId);
2616                 return false;
2617             }
2618         }
2619
2620         return true;
2621     }
2622
2623     private void programRemoteFibForBgpRoutes(final BigInteger remoteDpnId, final long vpnId,
2624             final VrfEntry vrfEntry, WriteTransaction tx, String rd, List<AdjacencyResult> adjacencyResults) {
2625         Preconditions.checkArgument(vrfEntry.getRoutePaths().size() <= 2);
2626
2627         if (adjacencyResults.size() == 1) {
2628             programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
2629             return;
2630         }
2631         // ECMP Use case, point to LB group. Move the mpls label accordingly.
2632         List<String> tunnelList =
2633                 adjacencyResults.stream()
2634                 .map(adjacencyResult -> adjacencyResult.getNextHopIp())
2635                 .sorted().collect(toList());
2636         String lbGroupKey = FibUtil.getGreLbGroupKey(tunnelList);
2637         long groupId = nextHopManager.createNextHopPointer(lbGroupKey);
2638         int index = 0;
2639         List<ActionInfo> actionInfos = new ArrayList<>();
2640         for (AdjacencyResult adjResult : adjacencyResults) {
2641             String nextHopIp = adjResult.getNextHopIp();
2642             java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
2643             if (!optionalLabel.isPresent()) {
2644                 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
2645                 continue;
2646             }
2647             long label = optionalLabel.get();
2648
2649             actionInfos.add(new ActionRegLoad(index, FibConstants.NXM_REG_MAPPING.get(index++), 0,
2650                     31, label));
2651         }
2652         List<InstructionInfo> instructions = new ArrayList<>();
2653         actionInfos.add(new ActionGroup(index, groupId));
2654         instructions.add(new InstructionApplyActions(actionInfos));
2655         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
2656     }
2657 }