fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/BaseVrfEntryHandler.java

   1 /*
   2  * Copyright © 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static java.util.stream.Collectors.toList;
  11 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  12
  13 import com.google.common.base.Optional;
  14 import java.math.BigInteger;
  15 import java.net.Inet4Address;
  16 import java.net.InetAddress;
  17 import java.net.UnknownHostException;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.List;
  21 import javax.annotation.Nonnull;
  22 import javax.inject.Inject;
  23 import javax.inject.Singleton;
  24 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  25 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  26 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  27 import org.opendaylight.genius.mdsalutil.ActionInfo;
  28 import org.opendaylight.genius.mdsalutil.FlowEntity;
  29 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  30 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  31 import org.opendaylight.genius.mdsalutil.MatchInfo;
  32 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  33 import org.opendaylight.genius.mdsalutil.NwConstants;
  34 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationEth;
  35 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationIp;
  36 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  37 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  38 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  39 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  41 import org.opendaylight.genius.mdsalutil.actions.ActionSetIcmpType;
  42 import org.opendaylight.genius.mdsalutil.actions.ActionSetSourceIp;
  43 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  44 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetDestination;
  47 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  48 import org.opendaylight.genius.mdsalutil.matches.MatchIcmpv4;
  49 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  50 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  51 import org.opendaylight.genius.mdsalutil.matches.MatchIpv6Destination;
  52 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  53 import org.opendaylight.genius.utils.batching.SubTransaction;
  54 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
  55 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  56 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  57 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  58 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroutes;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.Vpn;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.VpnKey;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutes;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutesKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.RoutesKey;
  82 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  83 import org.slf4j.Logger;
  84 import org.slf4j.LoggerFactory;
  85
  86
  87 @Singleton
  88 public class BaseVrfEntryHandler implements AutoCloseable {
  89
  90     private static final Logger LOG = LoggerFactory.getLogger(BaseVrfEntryHandler.class);
  91     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
  92     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
  93
  94     private final DataBroker dataBroker;
  95     private final NexthopManager nextHopManager;
  96     private final IMdsalApiManager mdsalManager;
  97     private final FibUtil fibUtil;
  98
  99     @Inject
 100     public BaseVrfEntryHandler(final DataBroker dataBroker,
 101                                final NexthopManager nexthopManager,
 102                                final IMdsalApiManager mdsalManager,
 103                                final FibUtil fibUtil) {
 104         this.dataBroker = dataBroker;
 105         this.nextHopManager = nexthopManager;
 106         this.mdsalManager = mdsalManager;
 107         this.fibUtil = fibUtil;
 108     }
 109
 110     @Override
 111     public void close() {
 112         LOG.info("{} closed", getClass().getSimpleName());
 113     }
 114
 115     protected FibUtil getFibUtil() {
 116         return fibUtil;
 117     }
 118
 119     protected NexthopManager getNextHopManager() {
 120         return nextHopManager;
 121     }
 122
 123     private void addAdjacencyResultToList(List<AdjacencyResult> adjacencyList, AdjacencyResult adjacencyResult) {
 124         if (adjacencyResult != null && !adjacencyList.contains(adjacencyResult)) {
 125             adjacencyList.add(adjacencyResult);
 126         }
 127     }
 128
 129     protected void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress,
 130                               final String ipPrefixAddress) {
 131         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}", dpId, vpnId, ipAddress);
 132         try {
 133             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress);
 134         } catch (NullPointerException e) {
 135             LOG.trace("", e);
 136         }
 137     }
 138
 139     @Nonnull
 140     protected List<AdjacencyResult> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
 141                                                      final VrfEntry vrfEntry, String rd) {
 142         List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
 143         FibHelper.sortIpAddress(routePaths);
 144         List<AdjacencyResult> adjacencyList = new ArrayList<>();
 145         List<String> prefixIpList;
 146         LOG.trace("resolveAdjacency called with remotedDpnId {}, vpnId{}, VrfEntry {}",
 147                 remoteDpnId, vpnId, vrfEntry);
 148         try {
 149             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 150                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
 151                 List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 152                         fibUtil.getVpnNameFromId(vpnId), usedRds, vrfEntry.getDestPrefix());
 153                 if (vpnExtraRoutes.isEmpty()) {
 154                     Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
 155                     // We don't want to provide an adjacencyList for an extra-route-prefix.
 156                     if (prefixInfo == null) {
 157                         LOG.debug("The extra route {} in rd {} for vpn {} has been removed from all the next hops",
 158                                 vrfEntry.getDestPrefix(), rd, vpnId);
 159                         return adjacencyList;
 160                     }
 161                     prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
 162                 } else {
 163                     List<String> prefixIpListLocal = new ArrayList<>();
 164                     vpnExtraRoutes.stream().forEach(route -> {
 165                         route.getNexthopIpList().stream().forEach(extraRouteIp -> {
 166                             String ipPrefix;
 167                             if (isIpv4Address(extraRouteIp)) {
 168                                 ipPrefix = extraRouteIp + NwConstants.IPV4PREFIX;
 169                             } else {
 170                                 ipPrefix = extraRouteIp + NwConstants.IPV6PREFIX;
 171                             }
 172                             prefixIpListLocal.add(ipPrefix);
 173                         });
 174                     });
 175                     prefixIpList = prefixIpListLocal;
 176                 }
 177             } else {
 178                 prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
 179             }
 180
 181             for (String prefixIp : prefixIpList) {
 182                 if (routePaths == null || routePaths.isEmpty()) {
 183                     LOG.trace("Processing Destination IP {} without NextHop IP", prefixIp);
 184                     AdjacencyResult adjacencyResult = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
 185                             prefixIp, null);
 186                     addAdjacencyResultToList(adjacencyList, adjacencyResult);
 187                     continue;
 188                 }
 189                 adjacencyList.addAll(routePaths.stream()
 190                         .map(routePath -> {
 191                             LOG.debug("NextHop IP for destination {} is {}", prefixIp,
 192                                     routePath.getNexthopAddress());
 193                             return nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
 194                                     prefixIp, routePath.getNexthopAddress());
 195                         })
 196                         .filter(adjacencyResult -> adjacencyResult != null && !adjacencyList.contains(adjacencyResult))
 197                         .distinct()
 198                         .collect(toList()));
 199             }
 200         } catch (NullPointerException e) {
 201             LOG.trace("", e);
 202         }
 203         return adjacencyList;
 204     }
 205
 206     protected void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
 207                             List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx,
 208                             List<SubTransaction> subTxns) {
 209         Boolean wrTxPresent = true;
 210         if (tx == null) {
 211             wrTxPresent = false;
 212             tx = dataBroker.newWriteOnlyTransaction();
 213         }
 214
 215         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
 216         String[] values = vrfEntry.getDestPrefix().split("/");
 217         String ipAddress = values[0];
 218         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 219         if (addOrRemove == NwConstants.ADD_FLOW) {
 220             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
 221         } else {
 222             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
 223         }
 224         InetAddress destPrefix;
 225         try {
 226             destPrefix = InetAddress.getByName(ipAddress);
 227         } catch (UnknownHostException e) {
 228             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 229                     vrfEntry.getDestPrefix(), rd, vpnId, dpId, e);
 230             return;
 231         }
 232
 233         List<MatchInfo> matches = new ArrayList<>();
 234
 235         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 236
 237         if (destPrefix instanceof Inet4Address) {
 238             matches.add(MatchEthernetType.IPV4);
 239             if (prefixLength != 0) {
 240                 matches.add(new MatchIpv4Destination(destPrefix.getHostAddress(), Integer.toString(prefixLength)));
 241             }
 242         } else {
 243             matches.add(MatchEthernetType.IPV6);
 244             if (prefixLength != 0) {
 245                 matches.add(new MatchIpv6Destination(destPrefix.getHostAddress() + "/" + prefixLength));
 246             }
 247         }
 248
 249         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 250         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
 251         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority,
 252                 flowRef, 0, 0,
 253                 COOKIE_VM_FIB_TABLE, matches, instructions);
 254         Flow flow = flowEntity.getFlowBuilder().build();
 255         String flowId = flowEntity.getFlowId();
 256         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 257         Node nodeDpn = FibUtil.buildDpnNode(dpId);
 258
 259         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 260                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 261                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 262
 263         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 264             SubTransaction subTransaction = new SubTransactionImpl();
 265             if (addOrRemove == NwConstants.ADD_FLOW) {
 266                 subTransaction.setInstanceIdentifier(flowInstanceId);
 267                 subTransaction.setInstance(flow);
 268                 subTransaction.setAction(SubTransaction.CREATE);
 269             } else {
 270                 subTransaction.setInstanceIdentifier(flowInstanceId);
 271                 subTransaction.setAction(SubTransaction.DELETE);
 272             }
 273             subTxns.add(subTransaction);
 274         }
 275
 276         if (addOrRemove == NwConstants.ADD_FLOW) {
 277             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
 278         } else {
 279             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 280         }
 281
 282         if (!wrTxPresent) {
 283             tx.submit();
 284         }
 285     }
 286
 287     protected void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, Prefixes prefixInfo,
 288                                         List<ActionInfo> actionInfos) {
 289         if (vrfEntry.getMac() != null) {
 290             actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
 291                     new MacAddress(vrfEntry.getMac())));
 292             return;
 293         }
 294         if (prefixInfo == null) {
 295             prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
 296             //Checking PrefixtoInterface again as it is populated later in some cases
 297             if (prefixInfo == null) {
 298                 LOG.debug("No prefix info found for prefix {}", vrfEntry.getDestPrefix());
 299                 return;
 300             }
 301         }
 302         String ipPrefix = prefixInfo.getIpAddress();
 303         String ifName = prefixInfo.getVpnInterfaceName();
 304         if (ifName == null) {
 305             LOG.debug("Failed to get VPN interface for prefix {}", ipPrefix);
 306             return;
 307         }
 308         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 309         if (vpnName == null) {
 310             LOG.debug("Failed to get VPN name for vpnId {}", vpnId);
 311             return;
 312         }
 313         String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, ipPrefix);
 314         if (macAddress == null) {
 315             LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix);
 316             return;
 317         }
 318         actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(), new MacAddress(macAddress)));
 319     }
 320
 321     protected void addTunnelInterfaceActions(AdjacencyResult adjacencyResult, long vpnId, VrfEntry vrfEntry,
 322                                            List<ActionInfo> actionInfos, String rd) {
 323         Class<? extends TunnelTypeBase> tunnelType =
 324                 VpnExtraRouteHelper.getTunnelType(nextHopManager.getInterfaceManager(),
 325                         adjacencyResult.getInterfaceName());
 326         if (tunnelType == null) {
 327             LOG.debug("Tunnel type not found for vrfEntry {}", vrfEntry);
 328             return;
 329         }
 330         // TODO - For now have added routePath into adjacencyResult so that we know for which
 331         // routePath this result is built for. If this is not possible construct a map which does
 332         // the same.
 333         String nextHopIp = adjacencyResult.getNextHopIp();
 334         java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
 335         if (!optionalLabel.isPresent()) {
 336             LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
 337             return;
 338         }
 339         long label = optionalLabel.get();
 340         BigInteger tunnelId = null;
 341         Prefixes prefixInfo = null;
 342         // FIXME vxlan vni bit set is not working properly with OVS.need to
 343         // revisit
 344         if (tunnelType.equals(TunnelTypeVxlan.class)) {
 345             prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
 346             //For extra route, the prefixInfo is fetched from the primary adjacency
 347             if (prefixInfo == null) {
 348                 prefixInfo = fibUtil.getPrefixToInterface(vpnId, adjacencyResult.getPrefix());
 349             }
 350             // Internet VPN VNI will be used as tun_id for NAT use-cases
 351             if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
 352                 if (vrfEntry.getL3vni() != null && vrfEntry.getL3vni() != 0) {
 353                     tunnelId = BigInteger.valueOf(vrfEntry.getL3vni());
 354                 }
 355             } else {
 356                 if (fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(prefixInfo.getSubnetId(), vpnId,
 357                         rd)) {
 358                     java.util.Optional<Long> optionalVni = fibUtil.getVniForVxlanNetwork(prefixInfo.getSubnetId());
 359                     if (!optionalVni.isPresent()) {
 360                         LOG.error("VNI not found for nexthop {} vrfEntry {} with subnetId {}", nextHopIp,
 361                                 vrfEntry, prefixInfo.getSubnetId());
 362                         return;
 363                     }
 364                     tunnelId = BigInteger.valueOf(optionalVni.get());
 365                 } else {
 366                     tunnelId = BigInteger.valueOf(label);
 367                 }
 368             }
 369         } else {
 370             tunnelId = BigInteger.valueOf(label);
 371         }
 372         LOG.debug("adding set tunnel id action for label {}", label);
 373         actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
 374         addRewriteDstMacAction(vpnId, vrfEntry, prefixInfo, actionInfos);
 375     }
 376
 377     public void programRemoteFib(final BigInteger remoteDpnId, final long vpnId,
 378                                   final VrfEntry vrfEntry, WriteTransaction tx, String rd,
 379                                   List<AdjacencyResult> adjacencyResults,
 380                                   List<SubTransaction> subTxns) {
 381         List<InstructionInfo> instructions = new ArrayList<>();
 382         for (AdjacencyResult adjacencyResult : adjacencyResults) {
 383             List<ActionInfo> actionInfos = new ArrayList<>();
 384             String egressInterface = adjacencyResult.getInterfaceName();
 385             if (FibUtil.isTunnelInterface(adjacencyResult)) {
 386                 addTunnelInterfaceActions(adjacencyResult, vpnId, vrfEntry, actionInfos, rd);
 387             } else {
 388                 addRewriteDstMacAction(vpnId, vrfEntry, null, actionInfos);
 389             }
 390             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(egressInterface,
 391                     actionInfos.size());
 392             if (egressActions.isEmpty()) {
 393                 LOG.error(
 394                         "Failed to retrieve egress action for prefix {} route-paths {} interface {}. "
 395                                 + "Aborting remote FIB entry creation.",
 396                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
 397                 return;
 398             }
 399             actionInfos.addAll(egressActions);
 400             instructions.add(new InstructionApplyActions(actionInfos));
 401         }
 402         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx, subTxns);
 403     }
 404
 405     public boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
 406                                            VrfEntry vrfEntry, String rd,
 407                                            WriteTransaction tx, List<SubTransaction> subTxns) {
 408         boolean isRemoteRoute = true;
 409         if (localNextHopInfo != null) {
 410             isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId());
 411         }
 412         if (isRemoteRoute) {
 413             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx, subTxns);
 414             LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
 415             return true;
 416         } else {
 417             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}",
 418                     rd, vrfEntry.getDestPrefix(), remoteDpnId);
 419             return false;
 420         }
 421     }
 422
 423     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
 424                                   final long vpnId, final VrfTablesKey vrfTableKey,
 425                                   final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional,
 426                                   WriteTransaction tx) {
 427
 428         Boolean wrTxPresent = true;
 429         if (tx == null) {
 430             wrTxPresent = false;
 431             tx = dataBroker.newWriteOnlyTransaction();
 432         }
 433
 434         LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
 435                 vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
 436         String rd = vrfTableKey.getRouteDistinguisher();
 437
 438         if (localDpnId != null && localDpnId != BigInteger.ZERO) {
 439             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
 440             if (extraRouteOptional.isPresent()) {
 441                 nextHopManager.setupLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix(),
 442                         Collections.emptyList() /*listBucketInfo*/ , false);
 443             }
 444             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx, null);
 445             LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
 446             return;
 447         }
 448
 449         // below two reads are kept as is, until best way is found to identify dpnID
 450         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
 451         if (extraRouteOptional.isPresent()) {
 452             nextHopManager.setupLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix(),
 453                     Collections.emptyList() /*listBucketInfo*/ , false);
 454         } else {
 455             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx, null);
 456         }
 457         if (!wrTxPresent) {
 458             tx.submit();
 459         }
 460     }
 461
 462     public static InstanceIdentifier<Routes> getVpnToExtrarouteIdentifier(String vpnName, String vrfId,
 463                                                                     String ipPrefix) {
 464         return InstanceIdentifier.builder(VpnToExtraroutes.class)
 465                 .child(Vpn.class, new VpnKey(vpnName)).child(ExtraRoutes.class,
 466                         new ExtraRoutesKey(vrfId)).child(Routes.class, new RoutesKey(ipPrefix)).build();
 467     }
 468
 469     public Routes getVpnToExtraroute(Long vpnId, String vpnRd, String destPrefix) {
 470         String optVpnName = fibUtil.getVpnNameFromId(vpnId);
 471         if (optVpnName != null) {
 472             InstanceIdentifier<Routes> vpnExtraRoutesId = getVpnToExtrarouteIdentifier(
 473                     optVpnName, vpnRd, destPrefix);
 474             return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, vpnExtraRoutesId).orNull();
 475         }
 476         return null;
 477     }
 478
 479     public FlowEntity buildL3vpnGatewayFlow(BigInteger dpId, String gwMacAddress, long vpnId) {
 480         List<MatchInfo> mkMatches = new ArrayList<>();
 481         mkMatches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 482         mkMatches.add(new MatchEthernetDestination(new MacAddress(gwMacAddress)));
 483         List<InstructionInfo> mkInstructions = new ArrayList<>();
 484         mkInstructions.add(new InstructionGotoTable(NwConstants.L3_FIB_TABLE));
 485         String flowId = FibUtil.getL3VpnGatewayFlowRef(NwConstants.L3_GW_MAC_TABLE, dpId, vpnId, gwMacAddress);
 486         return MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_GW_MAC_TABLE,
 487                 flowId, 20, flowId, 0, 0, NwConstants.COOKIE_L3_GW_MAC_TABLE, mkMatches, mkInstructions);
 488     }
 489
 490     public void installPingResponderFlowEntry(BigInteger dpnId, long vpnId, String routerInternalIp,
 491                                               MacAddress routerMac, long label, int addOrRemove) {
 492
 493         List<MatchInfo> matches = new ArrayList<>();
 494         matches.add(MatchIpProtocol.ICMP);
 495         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 496         matches.add(new MatchIcmpv4((short) 8, (short) 0));
 497         matches.add(MatchEthernetType.IPV4);
 498         matches.add(new MatchIpv4Destination(routerInternalIp, "32"));
 499
 500         List<ActionInfo> actionsInfos = new ArrayList<>();
 501
 502         // Set Eth Src and Eth Dst
 503         actionsInfos.add(new ActionMoveSourceDestinationEth());
 504         actionsInfos.add(new ActionSetFieldEthernetSource(routerMac));
 505
 506         // Move Ip Src to Ip Dst
 507         actionsInfos.add(new ActionMoveSourceDestinationIp());
 508         actionsInfos.add(new ActionSetSourceIp(routerInternalIp, "32"));
 509
 510         // Set the ICMP type to 0 (echo reply)
 511         actionsInfos.add(new ActionSetIcmpType((short) 0));
 512
 513         actionsInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
 514
 515         actionsInfos.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
 516
 517         List<InstructionInfo> instructions = new ArrayList<>();
 518
 519         instructions.add(new InstructionApplyActions(actionsInfos));
 520
 521         int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH;
 522         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, label, priority);
 523
 524         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef,
 525                 0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions);
 526
 527         if (addOrRemove == NwConstants.ADD_FLOW) {
 528             mdsalManager.syncInstallFlow(flowEntity);
 529         } else {
 530             mdsalManager.syncRemoveFlow(flowEntity);
 531         }
 532     }
 533 }