Bump odlparent->6.0.0,mdsal->5.0.3
[netvirt.git] / fibmanager / impl / src / main / java / org / opendaylight / netvirt / fibmanager / BaseVrfEntryHandler.java
1 /*
2  * Copyright © 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8 package org.opendaylight.netvirt.fibmanager;
9
10 import static java.util.stream.Collectors.toList;
11 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
12
13 import com.google.common.base.Optional;
14 import java.net.Inet4Address;
15 import java.net.InetAddress;
16 import java.net.UnknownHostException;
17 import java.time.Duration;
18 import java.time.temporal.ChronoUnit;
19 import java.util.ArrayList;
20 import java.util.Collections;
21 import java.util.List;
22 import javax.inject.Inject;
23 import javax.inject.Singleton;
24 import org.eclipse.jdt.annotation.NonNull;
25 import org.eclipse.jdt.annotation.Nullable;
26 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
27 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
28 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
29 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
30 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
31 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
32 import org.opendaylight.genius.mdsalutil.ActionInfo;
33 import org.opendaylight.genius.mdsalutil.FlowEntity;
34 import org.opendaylight.genius.mdsalutil.InstructionInfo;
35 import org.opendaylight.genius.mdsalutil.MDSALUtil;
36 import org.opendaylight.genius.mdsalutil.MatchInfo;
37 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
38 import org.opendaylight.genius.mdsalutil.NwConstants;
39 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationEth;
40 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationIp;
41 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
42 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
43 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
44 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
45 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
46 import org.opendaylight.genius.mdsalutil.actions.ActionSetIcmpType;
47 import org.opendaylight.genius.mdsalutil.actions.ActionSetSourceIp;
48 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
49 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
50 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
51 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetDestination;
52 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
53 import org.opendaylight.genius.mdsalutil.matches.MatchIcmpv4;
54 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
55 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
56 import org.opendaylight.genius.mdsalutil.matches.MatchIpv6Destination;
57 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
58 import org.opendaylight.genius.utils.batching.SubTransaction;
59 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
60 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
61 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
62 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
63 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
64 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
65 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
66 import org.opendaylight.serviceutils.upgrade.UpgradeState;
67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
68 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
70 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
74 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
78 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
79 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroutes;
85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.Vpn;
88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.VpnKey;
89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutes;
90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutesKey;
91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.RoutesKey;
93 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
94 import org.opendaylight.yangtools.yang.common.Uint32;
95 import org.opendaylight.yangtools.yang.common.Uint64;
96 import org.slf4j.Logger;
97 import org.slf4j.LoggerFactory;
98
99 @Singleton
100 public class BaseVrfEntryHandler implements AutoCloseable {
101
102     private static final Logger LOG = LoggerFactory.getLogger(BaseVrfEntryHandler.class);
103     private static final Uint64 COOKIE_VM_FIB_TABLE =  Uint64.valueOf("8000003", 16).intern();
104     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
105
106     private final DataBroker dataBroker;
107     private final ManagedNewTransactionRunner txRunner;
108     private final NexthopManager nextHopManager;
109     private final IMdsalApiManager mdsalManager;
110     private final FibUtil fibUtil;
111     private final UpgradeState upgradeState;
112     private final DataTreeEventCallbackRegistrar eventCallbacks;
113
114     @Inject
115     public BaseVrfEntryHandler(final DataBroker dataBroker,
116                                final NexthopManager nexthopManager,
117                                final IMdsalApiManager mdsalManager,
118                                final FibUtil fibUtil,
119                                final UpgradeState upgradeState,
120                                final DataTreeEventCallbackRegistrar eventCallbacks) {
121         this.dataBroker = dataBroker;
122         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
123         this.nextHopManager = nexthopManager;
124         this.mdsalManager = mdsalManager;
125         this.fibUtil = fibUtil;
126         this.upgradeState = upgradeState;
127         this.eventCallbacks = eventCallbacks;
128     }
129
130     @Override
131     public void close() {
132         LOG.info("{} closed", getClass().getSimpleName());
133     }
134
135     protected FibUtil getFibUtil() {
136         return fibUtil;
137     }
138
139     protected NexthopManager getNextHopManager() {
140         return nextHopManager;
141     }
142
143     private void addAdjacencyResultToList(List<AdjacencyResult> adjacencyList, AdjacencyResult adjacencyResult) {
144         if (adjacencyResult != null && !adjacencyList.contains(adjacencyResult)) {
145             adjacencyList.add(adjacencyResult);
146         }
147     }
148
149     protected void deleteLocalAdjacency(final Uint64 dpId, final Uint32 vpnId, final String ipAddress,
150                               final String ipPrefixAddress) {
151         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, primaryIpAddress {} currIpPrefix {}",
152                 dpId, vpnId, ipAddress, ipPrefixAddress);
153         try {
154             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress);
155         } catch (NullPointerException e) {
156             // FIXME: NPEs should not be caught but rather their root cause should be eliminated
157             LOG.trace("Failed to remove nexthop", e);
158         }
159     }
160
161     @NonNull
162     protected List<AdjacencyResult> resolveAdjacency(final Uint64 remoteDpnId, final Uint32 vpnId,
163                                                      final VrfEntry vrfEntry, String rd) {
164         List<RoutePaths> routePaths = new ArrayList<>(vrfEntry.nonnullRoutePaths());
165         FibHelper.sortIpAddress(routePaths);
166         List<AdjacencyResult> adjacencyList = new ArrayList<>();
167         List<String> prefixIpList;
168         LOG.trace("resolveAdjacency called with remotedDpnId {}, vpnId{}, VrfEntry {}",
169                 remoteDpnId, vpnId, vrfEntry);
170         final Class<? extends TunnelTypeBase> tunnelType;
171         try {
172             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
173                 tunnelType = TunnelTypeVxlan.class;
174                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
175                 List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
176                         fibUtil.getVpnNameFromId(vpnId), usedRds, vrfEntry.getDestPrefix());
177                 if (vpnExtraRoutes.isEmpty()) {
178                     Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
179                     /* We don't want to provide an adjacencyList for
180                      * (1) an extra-route-prefix or,
181                      * (2) for a local route without prefix-to-interface.
182                      * Allow only self-imported routes in such cases */
183                     if (prefixInfo == null && FibHelper
184                             .isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
185                         LOG.debug("The prefix {} in rd {} for vpn {} does not have a valid extra-route or"
186                                 + " prefix-to-interface entry in the data-store", vrfEntry.getDestPrefix(), rd, vpnId);
187                         return adjacencyList;
188                     }
189                     prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
190                 } else {
191                     List<String> prefixIpListLocal = new ArrayList<>();
192                     vpnExtraRoutes.stream().filter(route -> route.getNexthopIpList() != null).forEach(
193                         route -> route.getNexthopIpList().forEach(extraRouteIp -> {
194                             String ipPrefix;
195                             if (isIpv4Address(extraRouteIp)) {
196                                 ipPrefix = extraRouteIp + NwConstants.IPV4PREFIX;
197                             } else {
198                                 ipPrefix = extraRouteIp + NwConstants.IPV6PREFIX;
199                             }
200                             prefixIpListLocal.add(ipPrefix);
201                         }));
202                     prefixIpList = prefixIpListLocal;
203                 }
204             } else {
205                 prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
206                 if (vrfEntry.getEncapType() == VrfEntry.EncapType.Mplsgre) {
207                     tunnelType = TunnelTypeMplsOverGre.class;
208                 } else {
209                     tunnelType = TunnelTypeVxlan.class;
210                 }
211             }
212
213             for (String prefixIp : prefixIpList) {
214                 if (routePaths == null || routePaths.isEmpty()) {
215                     LOG.trace("Processing Destination IP {} without NextHop IP", prefixIp);
216                     AdjacencyResult adjacencyResult = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
217                             prefixIp, null, tunnelType);
218                     addAdjacencyResultToList(adjacencyList, adjacencyResult);
219                     continue;
220                 }
221                 adjacencyList.addAll(routePaths.stream()
222                         .map(routePath -> {
223                             LOG.debug("NextHop IP for destination {} is {}", prefixIp,
224                                     routePath.getNexthopAddress());
225                             return nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
226                                     prefixIp, routePath.getNexthopAddress(), tunnelType);
227                         })
228                         .filter(adjacencyResult -> adjacencyResult != null && !adjacencyList.contains(adjacencyResult))
229                         .distinct()
230                         .collect(toList()));
231             }
232         } catch (NullPointerException e) {
233             // FIXME: NPEs should not be caught but rather their root cause should be eliminated
234             LOG.trace("Failed to remove adjacency", e);
235         }
236         return adjacencyList;
237     }
238
239     protected void makeConnectedRoute(Uint64 dpId, Uint32 vpnId, VrfEntry vrfEntry, String rd,
240                                       @Nullable List<InstructionInfo> instructions, int addOrRemove,
241                                       WriteTransaction tx, @Nullable List<SubTransaction> subTxns) {
242         if (tx == null) {
243             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
244                 newTx -> makeConnectedRoute(dpId, vpnId, vrfEntry, rd, instructions, addOrRemove, newTx, subTxns)),
245                 LOG, "Error making connected route");
246             return;
247         }
248
249         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
250         String[] values = vrfEntry.getDestPrefix().split("/");
251         String ipAddress = values[0];
252         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
253         if (addOrRemove == NwConstants.ADD_FLOW) {
254             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
255         } else {
256             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
257         }
258         InetAddress destPrefix;
259         try {
260             destPrefix = InetAddress.getByName(ipAddress);
261         } catch (UnknownHostException e) {
262             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
263                     vrfEntry.getDestPrefix(), rd, vpnId, dpId, e);
264             return;
265         }
266
267         List<MatchInfo> matches = new ArrayList<>();
268
269         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId.longValue()),
270             MetaDataUtil.METADATA_MASK_VRFID));
271
272         if (destPrefix instanceof Inet4Address) {
273             matches.add(MatchEthernetType.IPV4);
274             if (prefixLength != 0) {
275                 matches.add(new MatchIpv4Destination(destPrefix.getHostAddress(), Integer.toString(prefixLength)));
276             }
277         } else {
278             matches.add(MatchEthernetType.IPV6);
279             if (prefixLength != 0) {
280                 matches.add(new MatchIpv6Destination(destPrefix.getHostAddress() + "/" + prefixLength));
281             }
282         }
283
284         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
285         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
286         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority,
287                 flowRef, 0, 0,
288                 COOKIE_VM_FIB_TABLE, matches, instructions);
289         Flow flow = flowEntity.getFlowBuilder().build();
290         String flowId = flowEntity.getFlowId();
291         FlowKey flowKey = new FlowKey(new FlowId(flowId));
292         Node nodeDpn = FibUtil.buildDpnNode(dpId);
293
294         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
295                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
296                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
297
298         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
299             SubTransaction subTransaction = new SubTransactionImpl();
300             if (addOrRemove == NwConstants.ADD_FLOW) {
301                 subTransaction.setInstanceIdentifier(flowInstanceId);
302                 subTransaction.setInstance(flow);
303                 subTransaction.setAction(SubTransaction.CREATE);
304             } else {
305                 subTransaction.setInstanceIdentifier(flowInstanceId);
306                 subTransaction.setAction(SubTransaction.DELETE);
307             }
308             subTxns.add(subTransaction);
309         }
310
311         if (addOrRemove == NwConstants.ADD_FLOW) {
312             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
313         } else {
314             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
315         }
316     }
317
318     protected void addRewriteDstMacAction(Uint32 vpnId, VrfEntry vrfEntry, @Nullable Prefixes prefixInfo,
319                                           List<ActionInfo> actionInfos) {
320         if (vrfEntry.getMac() != null) {
321             actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
322                     new MacAddress(vrfEntry.getMac())));
323             return;
324         }
325         if (prefixInfo == null) {
326             prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
327             //Checking PrefixtoInterface again as it is populated later in some cases
328             if (prefixInfo == null) {
329                 LOG.debug("No prefix info found for prefix {}", vrfEntry.getDestPrefix());
330                 return;
331             }
332         }
333         String ipPrefix = prefixInfo.getIpAddress();
334         String ifName = prefixInfo.getVpnInterfaceName();
335         if (ifName == null) {
336             LOG.debug("Failed to get VPN interface for prefix {}", ipPrefix);
337             return;
338         }
339         String vpnName = fibUtil.getVpnNameFromId(vpnId);
340         if (vpnName == null) {
341             LOG.debug("Failed to get VPN name for vpnId {}", vpnId);
342             return;
343         }
344         String macAddress = null;
345         if (vrfEntry.getParentVpnRd() != null) {
346             // Handling iRT/eRT use-case for missing destination mac address in Remote FIB flow
347             Optional<VpnInstanceOpDataEntry> vpnInstance = fibUtil.getVpnInstanceOpData(vrfEntry.getParentVpnRd());
348             if (vpnInstance.isPresent()) {
349                 macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnInstance.get().getVpnInstanceName(), ipPrefix);
350             } else {
351                 LOG.warn("VpnInstance missing for Parent Rd {} value for prefix {}", vrfEntry.getParentVpnRd(),
352                         vrfEntry.getDestPrefix());
353             }
354         } else {
355             macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, ipPrefix);
356         }
357         if (macAddress == null) {
358             LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix);
359             return;
360         }
361         actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(), new MacAddress(macAddress)));
362     }
363
364     protected void addTunnelInterfaceActions(AdjacencyResult adjacencyResult, Uint32 vpnId, VrfEntry vrfEntry,
365                                            List<ActionInfo> actionInfos, String rd) {
366         Class<? extends TunnelTypeBase> tunnelType =
367                 VpnExtraRouteHelper.getTunnelType(nextHopManager.getItmManager(), adjacencyResult.getInterfaceName());
368         if (tunnelType == null) {
369             LOG.debug("Tunnel type not found for vrfEntry {}", vrfEntry);
370             return;
371         }
372         // TODO - For now have added routePath into adjacencyResult so that we know for which
373         // routePath this result is built for. If this is not possible construct a map which does
374         // the same.
375         String nextHopIp = adjacencyResult.getNextHopIp();
376         java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
377         if (!optionalLabel.isPresent()) {
378             LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
379             return;
380         }
381         Uint32 label = optionalLabel.get();
382         Uint64 tunnelId = null;
383         Prefixes prefixInfo = null;
384         // FIXME vxlan vni bit set is not working properly with OVS.need to
385         // revisit
386         if (tunnelType.equals(TunnelTypeVxlan.class)) {
387             if (FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
388                 prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
389                 //For extra route, the prefixInfo is fetched from the primary adjacency
390                 if (prefixInfo == null) {
391                     prefixInfo = fibUtil.getPrefixToInterface(vpnId, adjacencyResult.getPrefix());
392                 }
393             } else {
394                 //Imported Route. Get Prefix Info from parent RD
395                 VpnInstanceOpDataEntry parentVpn =  fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
396                 prefixInfo = fibUtil.getPrefixToInterface(parentVpn.getVpnId(), adjacencyResult.getPrefix());
397             }
398             // Internet VPN VNI will be used as tun_id for NAT use-cases
399             if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
400                 if (vrfEntry.getL3vni() != null && vrfEntry.getL3vni().toJava() != 0) {
401                     tunnelId = Uint64.valueOf(vrfEntry.getL3vni().longValue());
402                 }
403             } else {
404                 if (FibUtil.isVxlanNetwork(prefixInfo.getNetworkType())) {
405                     tunnelId = Uint64.valueOf(prefixInfo.getSegmentationId().longValue());
406                 } else {
407                     LOG.warn("Network is not of type VXLAN for prefix {}."
408                             + "Going with default Lport Tag.", prefixInfo.toString());
409                     tunnelId = Uint64.valueOf(label.longValue());
410                 }
411             }
412         } else {
413             tunnelId = Uint64.valueOf(label.longValue());
414         }
415         LOG.debug("adding set tunnel id action for label {}", label);
416         actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
417         addRewriteDstMacAction(vpnId, vrfEntry, prefixInfo, actionInfos);
418     }
419
420     @Nullable
421     private InstanceIdentifier<Interface> getFirstAbsentInterfaceStateIid(List<AdjacencyResult> adjacencyResults) {
422         InstanceIdentifier<Interface> res = null;
423         for (AdjacencyResult adjacencyResult : adjacencyResults) {
424             String interfaceName = adjacencyResult.getInterfaceName();
425             if (null == fibUtil.getInterfaceStateFromOperDS(interfaceName)) {
426                 res = FibUtil.buildStateInterfaceId(interfaceName);
427                 break;
428             }
429         }
430
431         return res;
432     }
433
434     public void programRemoteFib(final Uint64 remoteDpnId, final Uint32 vpnId,
435                                  final VrfEntry vrfEntry, WriteTransaction tx, String rd,
436                                  List<AdjacencyResult> adjacencyResults,
437                                  @Nullable List<SubTransaction> subTxns) {
438         if (upgradeState.isUpgradeInProgress()) {
439             InstanceIdentifier<Interface> absentInterfaceStateIid = getFirstAbsentInterfaceStateIid(adjacencyResults);
440             if (absentInterfaceStateIid != null) {
441                 LOG.info("programRemoteFib: interface state for {} not yet present, waiting...",
442                          absentInterfaceStateIid);
443                 eventCallbacks.onAddOrUpdate(LogicalDatastoreType.OPERATIONAL,
444                     absentInterfaceStateIid,
445                     (before, after) -> {
446                         LOG.info("programRemoteFib: waited for and got interface state {}", absentInterfaceStateIid);
447                         LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
448                             (wtx) -> programRemoteFib(remoteDpnId, vpnId, vrfEntry, wtx, rd, adjacencyResults, null)),
449                             LOG, "Failed to program remote FIB {}", absentInterfaceStateIid);
450                         return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
451                     },
452                     Duration.of(15, ChronoUnit.MINUTES),
453                     (iid) -> {
454                         LOG.error("programRemoteFib: timed out waiting for {}", absentInterfaceStateIid);
455                         LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
456                             (wtx) -> programRemoteFib(remoteDpnId, vpnId, vrfEntry, wtx, rd, adjacencyResults, null)),
457                             LOG, "Failed to program timed-out remote FIB {}", absentInterfaceStateIid);
458                     });
459                 return;
460             }
461         }
462
463         List<InstructionInfo> instructions = new ArrayList<>();
464         for (AdjacencyResult adjacencyResult : adjacencyResults) {
465             List<ActionInfo> actionInfos = new ArrayList<>();
466             String egressInterface = adjacencyResult.getInterfaceName();
467             if (FibUtil.isTunnelInterface(adjacencyResult)) {
468                 addTunnelInterfaceActions(adjacencyResult, vpnId, vrfEntry, actionInfos, rd);
469             } else {
470                 addRewriteDstMacAction(vpnId, vrfEntry, null, actionInfos);
471             }
472             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(egressInterface,
473                     actionInfos.size(), true, vpnId, vrfEntry.getDestPrefix());
474             if (egressActions.isEmpty()) {
475                 LOG.error(
476                         "Failed to retrieve egress action for prefix {} route-paths {} interface {}. "
477                                 + "Aborting remote FIB entry creation.",
478                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
479                 return;
480             }
481             actionInfos.addAll(egressActions);
482             instructions.add(new InstructionApplyActions(actionInfos));
483         }
484         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx, subTxns);
485     }
486
487     public boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, Uint64 remoteDpnId, Uint32 vpnId,
488                                           VrfEntry vrfEntry, String rd,
489                                           WriteTransaction tx, @Nullable List<SubTransaction> subTxns) {
490         boolean isRemoteRoute = true;
491         if (localNextHopInfo != null) {
492             isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId());
493         }
494         if (isRemoteRoute) {
495             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx, subTxns);
496             LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
497             return true;
498         } else {
499             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}",
500                     rd, vrfEntry.getDestPrefix(), remoteDpnId);
501             return false;
502         }
503     }
504
505     public void deleteRemoteRoute(@Nullable final Uint64 localDpnId, final Uint64 remoteDpnId,
506                                   final Uint32 vpnId, final VrfTablesKey vrfTableKey,
507                                   final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional,
508                                   @Nullable WriteTransaction tx) {
509         if (tx == null) {
510             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
511                 newTx -> deleteRemoteRoute(localDpnId, remoteDpnId, vpnId, vrfTableKey, vrfEntry,
512                         extraRouteOptional, newTx)), LOG, "Error deleting remote route");
513             return;
514         }
515
516         LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
517                 vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
518         String rd = vrfTableKey.getRouteDistinguisher();
519
520         if (localDpnId != null && !Uint64.ZERO.equals(localDpnId)) {
521             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
522             if (extraRouteOptional.isPresent()) {
523                 nextHopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
524             }
525             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx, null);
526             LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
527             return;
528         }
529
530         // below two reads are kept as is, until best way is found to identify dpnID
531         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
532         if (extraRouteOptional.isPresent()) {
533             nextHopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
534         } else {
535             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx, null);
536         }
537     }
538
539     public static InstanceIdentifier<Routes> getVpnToExtrarouteIdentifier(String vpnName, String vrfId,
540                                                                     String ipPrefix) {
541         return InstanceIdentifier.builder(VpnToExtraroutes.class)
542                 .child(Vpn.class, new VpnKey(vpnName)).child(ExtraRoutes.class,
543                         new ExtraRoutesKey(vrfId)).child(Routes.class, new RoutesKey(ipPrefix)).build();
544     }
545
546     @Nullable
547     public Routes getVpnToExtraroute(Uint32 vpnId, String vpnRd, String destPrefix) {
548         String optVpnName = fibUtil.getVpnNameFromId(vpnId);
549         if (optVpnName != null) {
550             InstanceIdentifier<Routes> vpnExtraRoutesId = getVpnToExtrarouteIdentifier(
551                     optVpnName, vpnRd, destPrefix);
552             return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, vpnExtraRoutesId).orNull();
553         }
554         return null;
555     }
556
557     public FlowEntity buildL3vpnGatewayFlow(Uint64 dpId, String gwMacAddress, Uint32 vpnId) {
558         List<MatchInfo> mkMatches = new ArrayList<>();
559         mkMatches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId.longValue()),
560             MetaDataUtil.METADATA_MASK_VRFID));
561         mkMatches.add(new MatchEthernetDestination(new MacAddress(gwMacAddress)));
562         List<InstructionInfo> mkInstructions = new ArrayList<>();
563         mkInstructions.add(new InstructionGotoTable(NwConstants.L3_FIB_TABLE));
564         String flowId = FibUtil.getL3VpnGatewayFlowRef(NwConstants.L3_GW_MAC_TABLE, dpId, vpnId, gwMacAddress);
565         return MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_GW_MAC_TABLE,
566                 flowId, 20, flowId, 0, 0, NwConstants.COOKIE_L3_GW_MAC_TABLE, mkMatches, mkInstructions);
567     }
568
569     public void installPingResponderFlowEntry(Uint64 dpnId, Uint32 vpnId, String routerInternalIp,
570                                               MacAddress routerMac, Uint32 label, int addOrRemove) {
571
572         List<MatchInfo> matches = new ArrayList<>();
573         matches.add(MatchIpProtocol.ICMP);
574         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId.longValue()),
575             MetaDataUtil.METADATA_MASK_VRFID));
576         matches.add(new MatchIcmpv4((short) 8, (short) 0));
577         matches.add(MatchEthernetType.IPV4);
578         matches.add(new MatchIpv4Destination(routerInternalIp, "32"));
579
580         List<ActionInfo> actionsInfos = new ArrayList<>();
581
582         // Set Eth Src and Eth Dst
583         actionsInfos.add(new ActionMoveSourceDestinationEth());
584         actionsInfos.add(new ActionSetFieldEthernetSource(routerMac));
585
586         // Move Ip Src to Ip Dst
587         actionsInfos.add(new ActionMoveSourceDestinationIp());
588         actionsInfos.add(new ActionSetSourceIp(routerInternalIp, "32"));
589
590         // Set the ICMP type to 0 (echo reply)
591         actionsInfos.add(new ActionSetIcmpType((short) 0));
592
593         actionsInfos.add(new ActionNxLoadInPort(Uint64.ZERO));
594
595         actionsInfos.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
596
597         List<InstructionInfo> instructions = new ArrayList<>();
598
599         instructions.add(new InstructionApplyActions(actionsInfos));
600
601         int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH;
602         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, label, priority);
603
604         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef,
605                 0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions);
606
607         if (addOrRemove == NwConstants.ADD_FLOW) {
608             mdsalManager.syncInstallFlow(flowEntity);
609         } else {
610             mdsalManager.syncRemoveFlow(flowEntity);
611         }
612     }
613 }