fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/BaseVrfEntryHandler.java

   1 /*
   2  * Copyright © 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static java.util.stream.Collectors.toList;
  11 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  12
  13 import com.google.common.base.Optional;
  14 import java.math.BigInteger;
  15 import java.net.Inet4Address;
  16 import java.net.InetAddress;
  17 import java.net.UnknownHostException;
  18 import java.time.Duration;
  19 import java.time.temporal.ChronoUnit;
  20 import java.util.ArrayList;
  21 import java.util.Collections;
  22 import java.util.List;
  23 import javax.annotation.Nonnull;
  24 import javax.annotation.Nullable;
  25 import javax.inject.Inject;
  26 import javax.inject.Singleton;
  27 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  28 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  29 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  30 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  31 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  32 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  33 import org.opendaylight.genius.mdsalutil.ActionInfo;
  34 import org.opendaylight.genius.mdsalutil.FlowEntity;
  35 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  36 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  37 import org.opendaylight.genius.mdsalutil.MatchInfo;
  38 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  39 import org.opendaylight.genius.mdsalutil.NwConstants;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationEth;
  41 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationIp;
  42 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  43 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  44 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  45 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionSetIcmpType;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionSetSourceIp;
  49 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  50 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  51 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  52 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetDestination;
  53 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  54 import org.opendaylight.genius.mdsalutil.matches.MatchIcmpv4;
  55 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  56 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  57 import org.opendaylight.genius.mdsalutil.matches.MatchIpv6Destination;
  58 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  59 import org.opendaylight.genius.utils.batching.SubTransaction;
  60 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
  61 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  62 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  63 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  64 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  65 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  66 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  68 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroutes;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.Vpn;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.VpnKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutes;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutesKey;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.RoutesKey;
  93 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  94 import org.slf4j.Logger;
  95 import org.slf4j.LoggerFactory;
  96
  97 @Singleton
  98 public class BaseVrfEntryHandler implements AutoCloseable {
  99
 100     private static final Logger LOG = LoggerFactory.getLogger(BaseVrfEntryHandler.class);
 101     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 102     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 103
 104     private final DataBroker dataBroker;
 105     private final ManagedNewTransactionRunner txRunner;
 106     private final NexthopManager nextHopManager;
 107     private final IMdsalApiManager mdsalManager;
 108     private final FibUtil fibUtil;
 109     private final UpgradeState upgradeState;
 110     private final DataTreeEventCallbackRegistrar eventCallbacks;
 111
 112     @Inject
 113     public BaseVrfEntryHandler(final DataBroker dataBroker,
 114                                final NexthopManager nexthopManager,
 115                                final IMdsalApiManager mdsalManager,
 116                                final FibUtil fibUtil,
 117                                final UpgradeState upgradeState,
 118                                final DataTreeEventCallbackRegistrar eventCallbacks) {
 119         this.dataBroker = dataBroker;
 120         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 121         this.nextHopManager = nexthopManager;
 122         this.mdsalManager = mdsalManager;
 123         this.fibUtil = fibUtil;
 124         this.upgradeState = upgradeState;
 125         this.eventCallbacks = eventCallbacks;
 126     }
 127
 128     @Override
 129     public void close() {
 130         LOG.info("{} closed", getClass().getSimpleName());
 131     }
 132
 133     protected FibUtil getFibUtil() {
 134         return fibUtil;
 135     }
 136
 137     protected NexthopManager getNextHopManager() {
 138         return nextHopManager;
 139     }
 140
 141     private void addAdjacencyResultToList(List<AdjacencyResult> adjacencyList, AdjacencyResult adjacencyResult) {
 142         if (adjacencyResult != null && !adjacencyList.contains(adjacencyResult)) {
 143             adjacencyList.add(adjacencyResult);
 144         }
 145     }
 146
 147     protected void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress,
 148                               final String ipPrefixAddress) {
 149         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, primaryIpAddress {} currIpPrefix {}",
 150                 dpId, vpnId, ipAddress, ipPrefixAddress);
 151         try {
 152             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress);
 153         } catch (NullPointerException e) {
 154             LOG.trace("", e);
 155         }
 156     }
 157
 158     @Nonnull
 159     protected List<AdjacencyResult> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
 160                                                      final VrfEntry vrfEntry, String rd) {
 161         List<RoutePaths> routePaths = new ArrayList<>(vrfEntry.nonnullRoutePaths());
 162         FibHelper.sortIpAddress(routePaths);
 163         List<AdjacencyResult> adjacencyList = new ArrayList<>();
 164         List<String> prefixIpList;
 165         LOG.trace("resolveAdjacency called with remotedDpnId {}, vpnId{}, VrfEntry {}",
 166                 remoteDpnId, vpnId, vrfEntry);
 167         final Class<? extends TunnelTypeBase> tunnelType;
 168         try {
 169             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 170                 tunnelType = TunnelTypeVxlan.class;
 171                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
 172                 List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 173                         fibUtil.getVpnNameFromId(vpnId), usedRds, vrfEntry.getDestPrefix());
 174                 if (vpnExtraRoutes.isEmpty()) {
 175                     Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
 176                     /* We don't want to provide an adjacencyList for
 177                      * (1) an extra-route-prefix or,
 178                      * (2) for a local route without prefix-to-interface.
 179                      * Allow only self-imported routes in such cases */
 180                     if (prefixInfo == null && FibHelper
 181                             .isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 182                         LOG.debug("The prefix {} in rd {} for vpn {} does not have a valid extra-route or"
 183                                 + " prefix-to-interface entry in the data-store", vrfEntry.getDestPrefix(), rd, vpnId);
 184                         return adjacencyList;
 185                     }
 186                     prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
 187                 } else {
 188                     List<String> prefixIpListLocal = new ArrayList<>();
 189                     vpnExtraRoutes.stream().filter(route -> route.getNexthopIpList() != null).forEach(
 190                         route -> route.getNexthopIpList().forEach(extraRouteIp -> {
 191                             String ipPrefix;
 192                             if (isIpv4Address(extraRouteIp)) {
 193                                 ipPrefix = extraRouteIp + NwConstants.IPV4PREFIX;
 194                             } else {
 195                                 ipPrefix = extraRouteIp + NwConstants.IPV6PREFIX;
 196                             }
 197                             prefixIpListLocal.add(ipPrefix);
 198                         }));
 199                     prefixIpList = prefixIpListLocal;
 200                 }
 201             } else {
 202                 prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
 203                 if (vrfEntry.getEncapType() == VrfEntry.EncapType.Mplsgre) {
 204                     tunnelType = TunnelTypeMplsOverGre.class;
 205                 } else {
 206                     tunnelType = TunnelTypeVxlan.class;
 207                 }
 208             }
 209
 210             for (String prefixIp : prefixIpList) {
 211                 if (routePaths == null || routePaths.isEmpty()) {
 212                     LOG.trace("Processing Destination IP {} without NextHop IP", prefixIp);
 213                     AdjacencyResult adjacencyResult = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
 214                             prefixIp, null, tunnelType);
 215                     addAdjacencyResultToList(adjacencyList, adjacencyResult);
 216                     continue;
 217                 }
 218                 adjacencyList.addAll(routePaths.stream()
 219                         .map(routePath -> {
 220                             LOG.debug("NextHop IP for destination {} is {}", prefixIp,
 221                                     routePath.getNexthopAddress());
 222                             return nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
 223                                     prefixIp, routePath.getNexthopAddress(), tunnelType);
 224                         })
 225                         .filter(adjacencyResult -> adjacencyResult != null && !adjacencyList.contains(adjacencyResult))
 226                         .distinct()
 227                         .collect(toList()));
 228             }
 229         } catch (NullPointerException e) {
 230             LOG.trace("", e);
 231         }
 232         return adjacencyList;
 233     }
 234
 235     protected void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
 236                                       @Nullable List<InstructionInfo> instructions, int addOrRemove,
 237                                       WriteTransaction tx, @Nullable List<SubTransaction> subTxns) {
 238         if (tx == null) {
 239             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 240                 newTx -> makeConnectedRoute(dpId, vpnId, vrfEntry, rd, instructions, addOrRemove, newTx, subTxns)),
 241                 LOG, "Error making connected route");
 242             return;
 243         }
 244
 245         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
 246         String[] values = vrfEntry.getDestPrefix().split("/");
 247         String ipAddress = values[0];
 248         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 249         if (addOrRemove == NwConstants.ADD_FLOW) {
 250             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
 251         } else {
 252             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
 253         }
 254         InetAddress destPrefix;
 255         try {
 256             destPrefix = InetAddress.getByName(ipAddress);
 257         } catch (UnknownHostException e) {
 258             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 259                     vrfEntry.getDestPrefix(), rd, vpnId, dpId, e);
 260             return;
 261         }
 262
 263         List<MatchInfo> matches = new ArrayList<>();
 264
 265         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 266
 267         if (destPrefix instanceof Inet4Address) {
 268             matches.add(MatchEthernetType.IPV4);
 269             if (prefixLength != 0) {
 270                 matches.add(new MatchIpv4Destination(destPrefix.getHostAddress(), Integer.toString(prefixLength)));
 271             }
 272         } else {
 273             matches.add(MatchEthernetType.IPV6);
 274             if (prefixLength != 0) {
 275                 matches.add(new MatchIpv6Destination(destPrefix.getHostAddress() + "/" + prefixLength));
 276             }
 277         }
 278
 279         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 280         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
 281         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority,
 282                 flowRef, 0, 0,
 283                 COOKIE_VM_FIB_TABLE, matches, instructions);
 284         Flow flow = flowEntity.getFlowBuilder().build();
 285         String flowId = flowEntity.getFlowId();
 286         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 287         Node nodeDpn = FibUtil.buildDpnNode(dpId);
 288
 289         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 290                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 291                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 292
 293         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 294             SubTransaction subTransaction = new SubTransactionImpl();
 295             if (addOrRemove == NwConstants.ADD_FLOW) {
 296                 subTransaction.setInstanceIdentifier(flowInstanceId);
 297                 subTransaction.setInstance(flow);
 298                 subTransaction.setAction(SubTransaction.CREATE);
 299             } else {
 300                 subTransaction.setInstanceIdentifier(flowInstanceId);
 301                 subTransaction.setAction(SubTransaction.DELETE);
 302             }
 303             subTxns.add(subTransaction);
 304         }
 305
 306         if (addOrRemove == NwConstants.ADD_FLOW) {
 307             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
 308         } else {
 309             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 310         }
 311     }
 312
 313     protected void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, @Nullable Prefixes prefixInfo,
 314                                           List<ActionInfo> actionInfos) {
 315         if (vrfEntry.getMac() != null) {
 316             actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
 317                     new MacAddress(vrfEntry.getMac())));
 318             return;
 319         }
 320         if (prefixInfo == null) {
 321             prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
 322             //Checking PrefixtoInterface again as it is populated later in some cases
 323             if (prefixInfo == null) {
 324                 LOG.debug("No prefix info found for prefix {}", vrfEntry.getDestPrefix());
 325                 return;
 326             }
 327         }
 328         String ipPrefix = prefixInfo.getIpAddress();
 329         String ifName = prefixInfo.getVpnInterfaceName();
 330         if (ifName == null) {
 331             LOG.debug("Failed to get VPN interface for prefix {}", ipPrefix);
 332             return;
 333         }
 334         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 335         if (vpnName == null) {
 336             LOG.debug("Failed to get VPN name for vpnId {}", vpnId);
 337             return;
 338         }
 339         String macAddress = null;
 340         if (vrfEntry.getParentVpnRd() != null) {
 341             // Handling iRT/eRT use-case for missing destination mac address in Remote FIB flow
 342             Optional<VpnInstanceOpDataEntry> vpnInstance = fibUtil.getVpnInstanceOpData(vrfEntry.getParentVpnRd());
 343             if (vpnInstance.isPresent()) {
 344                 macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnInstance.get().getVpnInstanceName(), ipPrefix);
 345             } else {
 346                 LOG.warn("VpnInstance missing for Parent Rd {} value for prefix {}", vrfEntry.getParentVpnRd(),
 347                         vrfEntry.getDestPrefix());
 348             }
 349         } else {
 350             macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, ipPrefix);
 351         }
 352         if (macAddress == null) {
 353             LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix);
 354             return;
 355         }
 356         actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(), new MacAddress(macAddress)));
 357     }
 358
 359     protected void addTunnelInterfaceActions(AdjacencyResult adjacencyResult, long vpnId, VrfEntry vrfEntry,
 360                                            List<ActionInfo> actionInfos, String rd) {
 361         Class<? extends TunnelTypeBase> tunnelType =
 362                 VpnExtraRouteHelper.getTunnelType(nextHopManager.getItmManager(), adjacencyResult.getInterfaceName());
 363         if (tunnelType == null) {
 364             LOG.debug("Tunnel type not found for vrfEntry {}", vrfEntry);
 365             return;
 366         }
 367         // TODO - For now have added routePath into adjacencyResult so that we know for which
 368         // routePath this result is built for. If this is not possible construct a map which does
 369         // the same.
 370         String nextHopIp = adjacencyResult.getNextHopIp();
 371         java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
 372         if (!optionalLabel.isPresent()) {
 373             LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
 374             return;
 375         }
 376         long label = optionalLabel.get();
 377         BigInteger tunnelId = null;
 378         Prefixes prefixInfo = null;
 379         // FIXME vxlan vni bit set is not working properly with OVS.need to
 380         // revisit
 381         if (tunnelType.equals(TunnelTypeVxlan.class)) {
 382             if (FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 383                 prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
 384                 //For extra route, the prefixInfo is fetched from the primary adjacency
 385                 if (prefixInfo == null) {
 386                     prefixInfo = fibUtil.getPrefixToInterface(vpnId, adjacencyResult.getPrefix());
 387                 }
 388             } else {
 389                 //Imported Route. Get Prefix Info from parent RD
 390                 VpnInstanceOpDataEntry parentVpn =  fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 391                 prefixInfo = fibUtil.getPrefixToInterface(parentVpn.getVpnId(), adjacencyResult.getPrefix());
 392             }
 393             // Internet VPN VNI will be used as tun_id for NAT use-cases
 394             if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
 395                 if (vrfEntry.getL3vni() != null && vrfEntry.getL3vni() != 0) {
 396                     tunnelId = BigInteger.valueOf(vrfEntry.getL3vni());
 397                 }
 398             } else {
 399                 if (FibUtil.isVxlanNetwork(prefixInfo.getNetworkType())) {
 400                     tunnelId = BigInteger.valueOf(prefixInfo.getSegmentationId());
 401                 } else {
 402                     LOG.warn("Network is not of type VXLAN for prefix {}."
 403                             + "Going with default Lport Tag.", prefixInfo.toString());
 404                     tunnelId = BigInteger.valueOf(label);
 405                 }
 406             }
 407         } else {
 408             tunnelId = BigInteger.valueOf(label);
 409         }
 410         LOG.debug("adding set tunnel id action for label {}", label);
 411         actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
 412         addRewriteDstMacAction(vpnId, vrfEntry, prefixInfo, actionInfos);
 413     }
 414
 415     @Nullable
 416     private InstanceIdentifier<Interface> getFirstAbsentInterfaceStateIid(List<AdjacencyResult> adjacencyResults) {
 417         InstanceIdentifier<Interface> res = null;
 418         for (AdjacencyResult adjacencyResult : adjacencyResults) {
 419             String interfaceName = adjacencyResult.getInterfaceName();
 420             if (null == fibUtil.getInterfaceStateFromOperDS(interfaceName)) {
 421                 res = fibUtil.buildStateInterfaceId(interfaceName);
 422                 break;
 423             }
 424         }
 425
 426         return res;
 427     }
 428
 429     public void programRemoteFib(final BigInteger remoteDpnId, final long vpnId,
 430                                  final VrfEntry vrfEntry, WriteTransaction tx, String rd,
 431                                  List<AdjacencyResult> adjacencyResults,
 432                                  @Nullable List<SubTransaction> subTxns) {
 433         if (upgradeState.isUpgradeInProgress()) {
 434             InstanceIdentifier<Interface> absentInterfaceStateIid = getFirstAbsentInterfaceStateIid(adjacencyResults);
 435             if (absentInterfaceStateIid != null) {
 436                 LOG.info("programRemoteFib: interface state for {} not yet present, waiting...",
 437                          absentInterfaceStateIid);
 438                 eventCallbacks.onAddOrUpdate(LogicalDatastoreType.OPERATIONAL,
 439                     absentInterfaceStateIid,
 440                     (before, after) -> {
 441                         LOG.info("programRemoteFib: waited for and got interface state {}", absentInterfaceStateIid);
 442                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 443                             (wtx) -> programRemoteFib(remoteDpnId, vpnId, vrfEntry, wtx, rd, adjacencyResults, null));
 444                         return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 445                     },
 446                     Duration.of(15, ChronoUnit.MINUTES),
 447                     (iid) -> {
 448                         LOG.error("programRemoteFib: timed out waiting for {}", absentInterfaceStateIid);
 449                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 450                             (wtx) -> programRemoteFib(remoteDpnId, vpnId, vrfEntry, wtx, rd, adjacencyResults, null));
 451                     });
 452                 return;
 453             }
 454         }
 455
 456         List<InstructionInfo> instructions = new ArrayList<>();
 457         for (AdjacencyResult adjacencyResult : adjacencyResults) {
 458             List<ActionInfo> actionInfos = new ArrayList<>();
 459             String egressInterface = adjacencyResult.getInterfaceName();
 460             if (FibUtil.isTunnelInterface(adjacencyResult)) {
 461                 addTunnelInterfaceActions(adjacencyResult, vpnId, vrfEntry, actionInfos, rd);
 462             } else {
 463                 addRewriteDstMacAction(vpnId, vrfEntry, null, actionInfos);
 464             }
 465             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(egressInterface,
 466                     actionInfos.size(), true);
 467             if (egressActions.isEmpty()) {
 468                 LOG.error(
 469                         "Failed to retrieve egress action for prefix {} route-paths {} interface {}. "
 470                                 + "Aborting remote FIB entry creation.",
 471                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
 472                 return;
 473             }
 474             actionInfos.addAll(egressActions);
 475             instructions.add(new InstructionApplyActions(actionInfos));
 476         }
 477         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx, subTxns);
 478     }
 479
 480     public boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
 481                                           VrfEntry vrfEntry, String rd,
 482                                           WriteTransaction tx, @Nullable List<SubTransaction> subTxns) {
 483         boolean isRemoteRoute = true;
 484         if (localNextHopInfo != null) {
 485             isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId());
 486         }
 487         if (isRemoteRoute) {
 488             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx, subTxns);
 489             LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
 490             return true;
 491         } else {
 492             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}",
 493                     rd, vrfEntry.getDestPrefix(), remoteDpnId);
 494             return false;
 495         }
 496     }
 497
 498     public void deleteRemoteRoute(@Nullable final BigInteger localDpnId, final BigInteger remoteDpnId,
 499                                   final long vpnId, final VrfTablesKey vrfTableKey,
 500                                   final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional,
 501                                   @Nullable WriteTransaction tx) {
 502         if (tx == null) {
 503             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 504                 newTx -> deleteRemoteRoute(localDpnId, remoteDpnId, vpnId, vrfTableKey, vrfEntry,
 505                         extraRouteOptional, newTx)), LOG, "Error deleting remote route");
 506             return;
 507         }
 508
 509         LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
 510                 vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
 511         String rd = vrfTableKey.getRouteDistinguisher();
 512
 513         if (localDpnId != null && !BigInteger.ZERO.equals(localDpnId)) {
 514             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
 515             if (extraRouteOptional.isPresent()) {
 516                 nextHopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
 517             }
 518             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx, null);
 519             LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
 520             return;
 521         }
 522
 523         // below two reads are kept as is, until best way is found to identify dpnID
 524         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
 525         if (extraRouteOptional.isPresent()) {
 526             nextHopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
 527         } else {
 528             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx, null);
 529         }
 530     }
 531
 532     public static InstanceIdentifier<Routes> getVpnToExtrarouteIdentifier(String vpnName, String vrfId,
 533                                                                     String ipPrefix) {
 534         return InstanceIdentifier.builder(VpnToExtraroutes.class)
 535                 .child(Vpn.class, new VpnKey(vpnName)).child(ExtraRoutes.class,
 536                         new ExtraRoutesKey(vrfId)).child(Routes.class, new RoutesKey(ipPrefix)).build();
 537     }
 538
 539     @Nullable
 540     public Routes getVpnToExtraroute(Long vpnId, String vpnRd, String destPrefix) {
 541         String optVpnName = fibUtil.getVpnNameFromId(vpnId);
 542         if (optVpnName != null) {
 543             InstanceIdentifier<Routes> vpnExtraRoutesId = getVpnToExtrarouteIdentifier(
 544                     optVpnName, vpnRd, destPrefix);
 545             return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, vpnExtraRoutesId).orNull();
 546         }
 547         return null;
 548     }
 549
 550     public FlowEntity buildL3vpnGatewayFlow(BigInteger dpId, String gwMacAddress, long vpnId) {
 551         List<MatchInfo> mkMatches = new ArrayList<>();
 552         mkMatches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 553         mkMatches.add(new MatchEthernetDestination(new MacAddress(gwMacAddress)));
 554         List<InstructionInfo> mkInstructions = new ArrayList<>();
 555         mkInstructions.add(new InstructionGotoTable(NwConstants.L3_FIB_TABLE));
 556         String flowId = FibUtil.getL3VpnGatewayFlowRef(NwConstants.L3_GW_MAC_TABLE, dpId, vpnId, gwMacAddress);
 557         return MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_GW_MAC_TABLE,
 558                 flowId, 20, flowId, 0, 0, NwConstants.COOKIE_L3_GW_MAC_TABLE, mkMatches, mkInstructions);
 559     }
 560
 561     public void installPingResponderFlowEntry(BigInteger dpnId, long vpnId, String routerInternalIp,
 562                                               MacAddress routerMac, long label, int addOrRemove) {
 563
 564         List<MatchInfo> matches = new ArrayList<>();
 565         matches.add(MatchIpProtocol.ICMP);
 566         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 567         matches.add(new MatchIcmpv4((short) 8, (short) 0));
 568         matches.add(MatchEthernetType.IPV4);
 569         matches.add(new MatchIpv4Destination(routerInternalIp, "32"));
 570
 571         List<ActionInfo> actionsInfos = new ArrayList<>();
 572
 573         // Set Eth Src and Eth Dst
 574         actionsInfos.add(new ActionMoveSourceDestinationEth());
 575         actionsInfos.add(new ActionSetFieldEthernetSource(routerMac));
 576
 577         // Move Ip Src to Ip Dst
 578         actionsInfos.add(new ActionMoveSourceDestinationIp());
 579         actionsInfos.add(new ActionSetSourceIp(routerInternalIp, "32"));
 580
 581         // Set the ICMP type to 0 (echo reply)
 582         actionsInfos.add(new ActionSetIcmpType((short) 0));
 583
 584         actionsInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
 585
 586         actionsInfos.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
 587
 588         List<InstructionInfo> instructions = new ArrayList<>();
 589
 590         instructions.add(new InstructionApplyActions(actionsInfos));
 591
 592         int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH;
 593         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, label, priority);
 594
 595         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef,
 596                 0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions);
 597
 598         if (addOrRemove == NwConstants.ADD_FLOW) {
 599             mdsalManager.syncInstallFlow(flowEntity);
 600         } else {
 601             mdsalManager.syncRemoveFlow(flowEntity);
 602         }
 603     }
 604 }