2 * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netvirt.natservice.internal;
10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
13 import com.google.common.base.Optional;
14 import com.google.common.util.concurrent.FutureCallback;
15 import com.google.common.util.concurrent.Futures;
16 import com.google.common.util.concurrent.ListenableFuture;
17 import com.google.common.util.concurrent.MoreExecutors;
18 import java.math.BigInteger;
19 import java.net.Inet6Address;
20 import java.net.InetAddress;
21 import java.net.UnknownHostException;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.Collection;
25 import java.util.Collections;
26 import java.util.HashMap;
27 import java.util.HashSet;
28 import java.util.List;
30 import java.util.Objects;
32 import java.util.concurrent.ExecutionException;
33 import java.util.concurrent.Future;
34 import javax.annotation.PostConstruct;
35 import javax.inject.Inject;
36 import javax.inject.Singleton;
37 import org.eclipse.jdt.annotation.NonNull;
38 import org.eclipse.jdt.annotation.Nullable;
39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
40 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
41 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
42 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
43 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
44 import org.opendaylight.genius.infra.Datastore.Configuration;
45 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
46 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
48 import org.opendaylight.genius.infra.TypedWriteTransaction;
49 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
50 import org.opendaylight.genius.mdsalutil.ActionInfo;
51 import org.opendaylight.genius.mdsalutil.BucketInfo;
52 import org.opendaylight.genius.mdsalutil.FlowEntity;
53 import org.opendaylight.genius.mdsalutil.GroupEntity;
54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
56 import org.opendaylight.genius.mdsalutil.MatchInfo;
57 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
58 import org.opendaylight.genius.mdsalutil.NwConstants;
59 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
62 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
63 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
64 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
65 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
66 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
67 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
68 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
69 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
70 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
71 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
72 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
73 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
74 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
75 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
76 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
77 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
78 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
79 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
80 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
81 import org.opendaylight.netvirt.elanmanager.api.IElanService;
82 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
83 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
84 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
85 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
86 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
98 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
144 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
145 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
146 import org.opendaylight.yangtools.yang.common.RpcResult;
147 import org.opendaylight.yangtools.yang.common.Uint32;
148 import org.opendaylight.yangtools.yang.common.Uint64;
149 import org.slf4j.Logger;
150 import org.slf4j.LoggerFactory;
154 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
155 private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
157 private static final Uint64 COOKIE_TUNNEL = Uint64.valueOf("9000000", 16).intern();
158 private static final Uint64 COOKIE_VM_LFIB_TABLE = Uint64.valueOf("8000022", 16).intern();
160 private final DataBroker dataBroker;
161 private final ManagedNewTransactionRunner txRunner;
162 private final IMdsalApiManager mdsalManager;
163 private final ItmRpcService itmManager;
164 private final OdlInterfaceRpcService odlInterfaceRpcService;
165 private final IdManagerService idManager;
166 private final NaptManager naptManager;
167 private final NAPTSwitchSelector naptSwitchSelector;
168 private final IBgpManager bgpManager;
169 private final VpnRpcService vpnService;
170 private final FibRpcService fibService;
171 private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
172 private final NaptEventHandler naptEventHandler;
173 private final NaptPacketInHandler naptPacketInHandler;
174 private final IFibManager fibManager;
175 private final IVpnManager vpnManager;
176 private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
177 private final NatMode natMode;
178 private final IElanService elanManager;
179 private final JobCoordinator coordinator;
180 private final IInterfaceManager interfaceManager;
181 private final NatOverVxlanUtil natOverVxlanUtil;
182 private final int snatPuntTimeout;
185 public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
186 final ItmRpcService itmManager,
187 final OdlInterfaceRpcService odlInterfaceRpcService,
188 final IdManagerService idManager,
189 final NaptManager naptManager,
190 final NAPTSwitchSelector naptSwitchSelector,
191 final IBgpManager bgpManager,
192 final VpnRpcService vpnService,
193 final FibRpcService fibService,
194 final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
195 final NaptEventHandler naptEventHandler,
196 final NaptPacketInHandler naptPacketInHandler,
197 final IFibManager fibManager,
198 final IVpnManager vpnManager,
199 final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
200 final NatserviceConfig config,
201 final IElanService elanManager,
202 final JobCoordinator coordinator,
203 final NatOverVxlanUtil natOverVxlanUtil,
204 final IInterfaceManager interfaceManager) {
205 super(Routers.class, ExternalRoutersListener.class);
206 this.dataBroker = dataBroker;
207 this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
208 this.mdsalManager = mdsalManager;
209 this.itmManager = itmManager;
210 this.odlInterfaceRpcService = odlInterfaceRpcService;
211 this.idManager = idManager;
212 this.naptManager = naptManager;
213 this.naptSwitchSelector = naptSwitchSelector;
214 this.bgpManager = bgpManager;
215 this.vpnService = vpnService;
216 this.fibService = fibService;
217 this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
218 this.naptEventHandler = naptEventHandler;
219 this.naptPacketInHandler = naptPacketInHandler;
220 this.fibManager = fibManager;
221 this.vpnManager = vpnManager;
222 this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
223 this.elanManager = elanManager;
224 this.coordinator = coordinator;
225 this.interfaceManager = interfaceManager;
226 this.natOverVxlanUtil = natOverVxlanUtil;
227 if (config != null) {
228 this.natMode = config.getNatMode();
229 this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
231 this.natMode = NatMode.Controller;
232 this.snatPuntTimeout = 0;
239 LOG.info("{} init", getClass().getSimpleName());
240 // This class handles ExternalRouters for Controller SNAT mode.
241 // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
242 if (natMode == NatMode.Controller) {
243 registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
244 NatUtil.createGroupIdPool(idManager);
249 protected InstanceIdentifier<Routers> getWildCardPath() {
250 return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
254 // TODO Clean up the exception handling
255 @SuppressWarnings("checkstyle:IllegalCatch")
256 protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
257 // Populate the router-id-name container
258 String routerName = routers.getRouterName();
259 LOG.info("add : external router event for {}", routerName);
260 Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
261 NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
262 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
264 if (routers.isEnableSnat()) {
265 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
266 () -> Collections.singletonList(
267 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
268 LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
269 Uint32 bgpVpnId = NatConstants.INVALID_ID;
270 if (bgpVpnUuid != null) {
271 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
273 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
274 // Allocate Primary Napt Switch for this router
275 Uint64 primarySwitchId = getPrimaryNaptSwitch(routerName);
276 if (primarySwitchId != null && !primarySwitchId.equals(Uint64.ZERO)) {
277 handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
280 )), NatConstants.NAT_DJC_MAX_RETRIES);
282 LOG.info("add : SNAT is disabled for external router {} ", routerName);
284 } catch (Exception ex) {
285 LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
290 public void handleEnableSnat(Routers routers, Uint32 routerId, Uint64 primarySwitchId, Uint32 bgpVpnId,
291 TypedWriteTransaction<Configuration> confTx) {
292 String routerName = routers.getRouterName();
293 LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
295 naptManager.initialiseExternalCounter(routers, routerId);
296 subnetRegisterMapping(routers, routerId);
298 LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
299 primarySwitchId, routerName);
301 ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
302 routers.getNetworkId());
303 if (extNwProvType == null) {
304 LOG.error("handleEnableSnat : External Network Provider Type missing");
308 if (bgpVpnId != NatConstants.INVALID_ID) {
309 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, confTx,
312 // write metadata and punt
313 installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
314 handlePrimaryNaptSwitch(primarySwitchId, routerName, routerId, confTx);
315 // Now install entries in SNAT tables to point to Primary for each router
316 List<Uint64> switches = naptSwitchSelector.getDpnsForVpn(routerName);
317 for (Uint64 dpnId : switches) {
318 // Handle switches and NAPT switches separately
319 if (!dpnId.equals(primarySwitchId)) {
320 LOG.debug("handleEnableSnat : Handle Ordinary switch");
321 handleSwitches(dpnId, routerName, routerId, primarySwitchId);
326 Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
327 if (externalIps.isEmpty()) {
328 LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
331 for (String externalIpAddrPrefix : externalIps) {
332 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
333 + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
334 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
338 LOG.debug("handleEnableSnat : Exit");
341 private Uint64 getPrimaryNaptSwitch(String routerName) {
342 // Allocate Primary Napt Switch for this router
343 Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
344 if (primarySwitchId != null && !primarySwitchId.equals(Uint64.ZERO)) {
345 LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch with DPN ID {} is already elected for router {}",
346 primarySwitchId, routerName);
347 return primarySwitchId;
349 return selectNewNAPTSwitch(routerName);
352 private Uint64 selectNewNAPTSwitch(String routerName) {
353 // Allocated an id from VNI pool for the Router.
354 natOverVxlanUtil.getRouterVni(routerName, NatConstants.INVALID_ID);
355 Uint64 primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName, null);
356 LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch DPN ID {}", primarySwitchId);
358 return primarySwitchId;
361 protected void installNaptPfibExternalOutputFlow(String routerName, Uint32 routerId, Uint64 dpnId,
362 TypedWriteTransaction<Configuration> confTx) {
363 Uint32 extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
364 if (extVpnId == NatConstants.INVALID_ID) {
365 LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
368 List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
369 if (externalIps.isEmpty()) {
370 LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
374 for (String ip : externalIps) {
375 Uuid subnetId = getSubnetIdForFixedIp(ip);
376 if (subnetId != null) {
377 Uint32 subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
378 if (subnetVpnId != NatConstants.INVALID_ID) {
379 extVpnId = subnetVpnId;
381 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
382 dpnId, extVpnId, subnetId);
383 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
384 if (postNaptFlowEntity != null) {
385 mdsalManager.addFlow(confTx, postNaptFlowEntity);
392 private Uuid getSubnetIdForFixedIp(String ip) {
394 IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
395 Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
396 return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
398 LOG.error("getSubnetIdForFixedIp : ip is null");
402 protected void subnetRegisterMapping(Routers routerEntry, Uint32 segmentId) {
403 LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
404 List<String> externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
406 int extIpCounter = externalIps.size();
407 LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
408 counter, extIpCounter);
409 @Nullable List<Uuid> subnetIds = routerEntry.getSubnetIds();
410 if (subnetIds == null) {
413 for (Uuid subnet : subnetIds) {
414 LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
415 InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
416 .builder(Subnetmaps.class)
417 .child(Subnetmap.class, new SubnetmapKey(subnet))
419 Optional<Subnetmap> sn;
421 sn = SingleTransactionDataBroker.syncReadOptional(dataBroker,
422 LogicalDatastoreType.CONFIGURATION, subnetmapId);
423 } catch (ReadFailedException e) {
424 LOG.error("Failed to read SubnetMap for subnetmap Id {}", subnetmapId, e);
425 sn = Optional.absent();
427 if (sn.isPresent()) {
429 Subnetmap subnetmapEntry = sn.get();
430 String subnetString = subnetmapEntry.getSubnetIp();
431 String[] subnetSplit = subnetString.split("/");
432 String subnetIp = subnetSplit[0];
434 InetAddress address = InetAddress.getByName(subnetIp);
435 if (address instanceof Inet6Address) {
436 LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
437 + "{} ", subnet, routerEntry.getRouterName(), address);
440 } catch (UnknownHostException e) {
441 LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
444 String subnetPrefix = "0";
445 if (subnetSplit.length == 2) {
446 subnetPrefix = subnetSplit[1];
448 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
449 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
450 subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
452 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
453 counter, extIpCounter);
454 if (extIpCounter != 0) {
455 if (counter < extIpCounter) {
456 String[] ipSplit = externalIps.get(counter).split("/");
457 String externalIp = ipSplit[0];
458 String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
459 if (ipSplit.length == 2) {
460 extPrefix = ipSplit[1];
462 IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
463 LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix is {}",
464 externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
465 naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
466 LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
467 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
469 counter = 0; //Reset the counter which runs on externalIps for round-robbin effect
470 LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
471 String[] ipSplit = externalIps.get(counter).split("/");
472 String externalIp = ipSplit[0];
473 String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
474 if (ipSplit.length == 2) {
475 extPrefix = ipSplit[1];
477 IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
478 LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix is {}",
479 externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
480 naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
481 LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
482 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
483 externalIp, extPrefix);
487 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
489 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
494 private void addOrDelDefFibRouteToSNAT(String routerName, Uint32 routerId, Uint32 bgpVpnId,
495 Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx)
496 throws ExecutionException, InterruptedException {
497 //Check if BGP VPN exists. If exists then invoke the new method.
498 if (bgpVpnId != NatConstants.INVALID_ID) {
499 if (bgpVpnUuid != null) {
500 String bgpVpnName = bgpVpnUuid.getValue();
501 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
502 bgpVpnId, bgpVpnName);
503 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
504 .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
505 confTx.put(getRoutersIdentifier(bgpVpnId), rtrs, CREATE_MISSING_PARENTS);
508 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
510 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
515 //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
516 addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
519 private void addOrDelDefaultFibRouteForSNAT(String routerName, Uint32 routerId, boolean create,
520 TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
521 List<Uint64> switches = naptSwitchSelector.getDpnsForVpn(routerName);
522 if (switches.isEmpty()) {
523 LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
526 if (routerId == NatConstants.INVALID_ID) {
527 LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
528 + "default NAT route in FIB", routerName);
531 for (Uint64 dpnId : switches) {
533 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
534 + "for the internal vpn-id {}", routerId, dpnId, routerId);
535 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
537 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
538 + "for the internal vpn-id {}", routerId, dpnId, routerId);
539 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
544 private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, Uint32 routerId,
545 Uint32 bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
546 List<Uint64> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
547 if (dpnIds.isEmpty()) {
548 LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
549 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
552 for (Uint64 dpnId : dpnIds) {
553 if (bgpVpnId != NatConstants.INVALID_ID) {
554 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
555 + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
556 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
558 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
559 + "in dpn {} for the internal vpn", routerId, dpnId);
560 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
565 private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, Uint32 routerId, Uint32 bgpVpnId,
566 TypedReadWriteTransaction<Configuration> confTx)
567 throws ExecutionException, InterruptedException {
568 List<Uint64> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
569 if (dpnIds.isEmpty()) {
570 LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
571 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
574 for (Uint64 dpnId : dpnIds) {
575 if (bgpVpnId != NatConstants.INVALID_ID) {
576 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
577 + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
578 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
580 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
581 + "in dpn {} for the internal vpn", routerId, dpnId);
582 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
587 protected void installOutboundMissEntry(String routerName, Uint32 routerId, Uint64 primarySwitchId,
588 TypedWriteTransaction<Configuration> confTx) {
589 LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
590 if (routerId != NatConstants.INVALID_ID) {
591 LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
592 primarySwitchId, routerId);
593 createOutboundTblEntry(primarySwitchId, routerId, confTx);
595 LOG.error("installOutboundMissEntry : Unable to fetch Router Id for RouterName {}, failed to "
596 + "createAndInstallMissEntry", routerName);
600 public String getFlowRefOutbound(Uint64 dpnId, short tableId, Uint32 routerID, int protocol) {
601 return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
602 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
605 private String getFlowRefNaptPreFib(Uint64 dpnId, short tableId, Uint32 vpnId) {
606 return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
607 .FLOWID_SEPARATOR + vpnId;
610 public Uint64 getCookieOutboundFlow(Uint32 routerId) {
611 return Uint64.valueOf((NwConstants.COOKIE_OUTBOUND_NAPT_TABLE).toJava().add(
612 new BigInteger("0110001", 16)).add(BigInteger.valueOf(routerId.longValue())));
615 private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, Uint64 cookie) {
618 int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
620 if (protocol == NwConstants.IP_PROT_TCP) {
621 l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
622 l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
624 l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
625 l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
627 List<ActionLearn.FlowMod> flowMods = Arrays.asList(
628 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
629 NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
630 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
631 NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
632 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
633 NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
634 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
635 NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
636 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
637 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
638 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
639 MetaDataUtil.METADATA_VPN_ID_OFFSET,
640 NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
641 MetaDataUtil.METADATA_VPN_ID_BITLEN));
643 return new ActionLearn(0, hardTimeout, 7, cookie, 0,
644 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
647 private FlowEntity buildIcmpDropFlow(Uint64 dpnId, Uint32 routerId, Uint32 vpnId) {
648 List<MatchInfo> matches = new ArrayList<>();
649 matches.add(MatchEthernetType.IPV4);
650 matches.add(MatchIpProtocol.ICMP);
651 matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId.longValue()),
652 MetaDataUtil.METADATA_MASK_VRFID));
654 List<ActionInfo> actionInfos = new ArrayList<>();
655 actionInfos.add(new ActionDrop());
657 List<InstructionInfo> instructions = new ArrayList<>();
658 instructions.add(new InstructionApplyActions(actionInfos));
660 String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
661 NwConstants.IP_PROT_ICMP);
662 FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
663 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
664 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
668 protected FlowEntity buildOutboundFlowEntity(Uint64 dpId, Uint32 routerId, int protocol) {
669 LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
670 Uint64 cookie = getCookieOutboundFlow(routerId);
671 List<MatchInfo> matches = new ArrayList<>();
672 matches.add(MatchEthernetType.IPV4);
673 matches.add(new MatchIpProtocol((short)protocol));
674 matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
675 MetaDataUtil.METADATA_MASK_VRFID));
677 List<InstructionInfo> instructions = new ArrayList<>();
678 List<ActionInfo> actionsInfos = new ArrayList<>();
679 actionsInfos.add(new ActionPuntToController());
680 if (snatPuntTimeout != 0) {
681 actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
683 instructions.add(new InstructionApplyActions(actionsInfos));
685 String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
686 FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
688 cookie, matches, instructions);
689 LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
693 public void createOutboundTblEntry(Uint64 dpnId, Uint32 routerId, TypedWriteTransaction<Configuration> confTx) {
694 LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
695 FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
696 LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
697 mdsalManager.addFlow(confTx, tcpflowEntity);
699 FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
700 LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
701 mdsalManager.addFlow(confTx, udpflowEntity);
703 FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
704 LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
705 mdsalManager.addFlow(confTx, icmpDropFlow);
709 protected String getTunnelInterfaceName(Uint64 srcDpId, Uint64 dstDpId) {
710 Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
711 RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
713 Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
714 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
715 .setSourceDpid(srcDpId)
716 .setDestinationDpid(dstDpId)
717 .setTunnelType(tunType)
719 rpcResult = result.get();
720 if (!rpcResult.isSuccessful()) {
721 tunType = TunnelTypeGre.class;
722 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
723 .setSourceDpid(srcDpId)
724 .setDestinationDpid(dstDpId)
725 .setTunnelType(tunType)
727 rpcResult = result.get();
728 if (!rpcResult.isSuccessful()) {
729 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
730 rpcResult.getErrors());
732 return rpcResult.getResult().getInterfaceName();
734 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
735 rpcResult.getErrors());
737 return rpcResult.getResult().getInterfaceName();
739 } catch (InterruptedException | ExecutionException | NullPointerException e) {
740 LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
741 + "between {} and {}", srcDpId, dstDpId, e);
747 protected void installSnatMissEntryForPrimrySwch(Uint64 dpnId, String routerName, Uint32 routerId,
748 TypedWriteTransaction<Configuration> confTx) {
750 LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
751 // Install miss entry pointing to group
752 FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
753 mdsalManager.addFlow(confTx, flowEntity);
756 protected void installSnatMissEntry(Uint64 dpnId, List<BucketInfo> bucketInfo,
757 String routerName, Uint32 routerId) {
758 LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
759 dpnId, bucketInfo.get(0));
760 // Install the select group
761 Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
762 NatUtil.getGroupIdKey(routerName));
763 if (groupId == NatConstants.INVALID_ID) {
764 LOG.error("installSnatMissEntry: Unable to obtain group ID for Key: {}", routerName);
767 GroupEntity groupEntity =
768 MDSALUtil.buildGroupEntity(dpnId, groupId.longValue(), routerName, GroupTypes.GroupAll, bucketInfo);
769 LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
770 mdsalManager.syncInstallGroup(groupEntity);
771 // Install miss entry pointing to group
772 FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
773 if (flowEntity == null) {
774 LOG.error("installSnatMissEntry : Flow entity received as NULL. "
775 + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
776 + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
779 mdsalManager.installFlow(flowEntity);
782 void installGroup(Uint64 dpnId, String routerName, Uint32 groupId, List<BucketInfo> bucketInfo) {
783 GroupEntity groupEntity =
784 MDSALUtil.buildGroupEntity(dpnId, groupId.longValue(), routerName, GroupTypes.GroupAll, bucketInfo);
785 LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
786 mdsalManager.syncInstallGroup(groupEntity);
789 private FlowEntity buildSnatFlowEntity(Uint64 dpId, String routerName, Uint32 routerId, Uint32 groupId) {
790 LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
791 dpId, routerName, groupId);
792 List<MatchInfo> matches = new ArrayList<>();
793 matches.add(MatchEthernetType.IPV4);
794 matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
795 MetaDataUtil.METADATA_MASK_VRFID));
797 List<ActionInfo> actionsInfo = new ArrayList<>();
798 Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager, idManager,
799 routerId, routerName);
800 actionsInfo.add(new ActionSetFieldTunnelId(tunnelId));
801 LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
802 actionsInfo.add(new ActionGroup(groupId.longValue()));
803 List<InstructionInfo> instructions = new ArrayList<>();
804 instructions.add(new InstructionApplyActions(actionsInfo));
805 String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
806 FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
807 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
808 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
810 LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
814 private FlowEntity buildSnatFlowEntityForPrmrySwtch(Uint64 dpId, String routerName, Uint32 routerId) {
816 LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
818 List<MatchInfo> matches = new ArrayList<>();
819 matches.add(MatchEthernetType.IPV4);
820 matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
821 MetaDataUtil.METADATA_MASK_VRFID));
823 List<InstructionInfo> instructions = new ArrayList<>();
824 instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
826 String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
827 FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
828 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
829 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
831 LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
835 // TODO : Replace this with ITM Rpc once its available with full functionality
836 protected void installTerminatingServiceTblEntry(Uint64 dpnId, String routerName, Uint32 routerId,
837 TypedWriteTransaction<Configuration> confTx) {
839 LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
841 FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
842 if (flowEntity == null) {
843 LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
844 + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
845 + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
849 mdsalManager.addFlow(confTx, flowEntity);
853 private FlowEntity buildTsFlowEntity(Uint64 dpId, String routerName, Uint32 routerId) {
854 List<MatchInfo> matches = new ArrayList<>();
855 matches.add(MatchEthernetType.IPV4);
856 Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager,
857 idManager, routerId, routerName);
858 matches.add(new MatchTunnelId(tunnelId));
859 String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, Uint32.valueOf(tunnelId.longValue()));
860 List<InstructionInfo> instructions = new ArrayList<>();
861 instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
862 MetaDataUtil.METADATA_MASK_VRFID));
863 instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
864 FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
865 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
866 NwConstants.COOKIE_TS_TABLE, matches, instructions);
870 public String getFlowRefTs(Uint64 dpnId, short tableId, Uint32 routerID) {
871 return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
872 .FLOWID_SEPARATOR + routerID;
875 public static String getFlowRefSnat(Uint64 dpnId, short tableId, String routerID) {
876 return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
877 .FLOWID_SEPARATOR + routerID;
880 protected void handleSwitches(Uint64 dpnId, String routerName, Uint32 routerId, Uint64 primarySwitchId) {
881 LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
882 List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
883 String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
884 List<BucketInfo> listBucketInfo = new ArrayList<>();
886 if (ifNamePrimary != null) {
887 LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
888 listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
889 interfaceManager, ifNamePrimary, routerId, true);
890 if (listActionInfoPrimary.isEmpty()) {
891 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
892 + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
896 LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
897 + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
899 BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
901 listBucketInfo.add(0, bucketPrimary);
902 installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
905 List<BucketInfo> getBucketInfoForNonNaptSwitches(Uint64 nonNaptSwitchId,
906 Uint64 primarySwitchId, String routerName, Uint32 routerId) {
907 List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
908 String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
909 List<BucketInfo> listBucketInfo = new ArrayList<>();
911 if (ifNamePrimary != null) {
912 LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
914 listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
915 interfaceManager, ifNamePrimary, routerId, true);
916 if (listActionInfoPrimary.isEmpty()) {
917 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
918 + "for router {} towards Napt-switch {} via tunnel interface {}",
919 nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
922 LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
923 + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
925 BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
927 listBucketInfo.add(0, bucketPrimary);
928 return listBucketInfo;
931 protected void handlePrimaryNaptSwitch(Uint64 dpnId, String routerName, Uint32 routerId,
932 TypedWriteTransaction<Configuration> confTx) {
935 * Primary NAPT Switch – bucket Should always point back to its own Outbound Table
938 LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
941 List<BucketInfo> listBucketInfo = new ArrayList<>();
942 List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
943 listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
944 BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
945 listBucketInfo.add(0, bucketPrimary);
948 installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
949 installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
950 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
951 installNaptPfibEntry(dpnId, routerId, confTx);
952 Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
953 if (networkId != null) {
954 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
955 if (vpnUuid != null) {
956 Uint32 vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
957 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
958 installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
959 //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
960 if (vpnId != NatConstants.INVALID_ID) {
961 installNaptPfibEntry(dpnId, vpnId, null);
963 return Collections.emptyList();
966 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
969 LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
973 public void installNaptPfibEntry(Uint64 dpnId, Uint32 segmentId,
974 @Nullable TypedWriteTransaction<Configuration> confTx) {
975 LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
976 FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
977 if (confTx != null) {
978 mdsalManager.addFlow(confTx, naptPfibFlowEntity);
980 mdsalManager.installFlow(naptPfibFlowEntity);
984 public FlowEntity buildNaptPfibFlowEntity(Uint64 dpId, Uint32 segmentId) {
986 LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
987 List<MatchInfo> matches = new ArrayList<>();
988 matches.add(MatchEthernetType.IPV4);
989 matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId.longValue()),
990 MetaDataUtil.METADATA_MASK_VRFID));
992 ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
993 ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
994 listActionInfo.add(new ActionNxLoadInPort(Uint64.valueOf(BigInteger.ZERO)));
995 listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
996 instructionInfo.add(new InstructionApplyActions(listActionInfo));
998 String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
999 FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
1000 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
1001 NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
1002 LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
1006 public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, Uint64 dpnId, Routers router,
1007 Uint32 routerId, String routerName, String externalIp) {
1008 LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1009 dpnId, routerId, externalIp);
1010 Uuid networkId = router.getNetworkId();
1011 if (networkId == null) {
1012 LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1015 final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1016 if (vpnName == null) {
1017 LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1018 + "configuration {} in router {}", networkId, externalIp, routerId);
1021 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1022 externalIp, networkId, router, confTx);
1023 LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1024 dpnId, routerId, externalIp);
1027 public void advToBgpAndInstallFibAndTsFlows(final Uint64 dpnId, final short tableId, final String vpnName,
1028 final Uint32 routerId, final String routerName, final String externalIp,
1029 final Uuid extNetworkId, @Nullable final Routers router,
1030 final TypedWriteTransaction<Configuration> confTx) {
1031 LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1032 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1033 String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1034 String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1035 if (rd == null || rd.isEmpty()) {
1036 LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1039 ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1040 if (extNwProvType == null) {
1041 LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1044 if (extNwProvType == ProviderTypes.VXLAN) {
1045 evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1046 nextHopIp, routerId, routerName, extNetworkId, confTx);
1049 //Generate VPN label for the external IP
1050 GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1051 .setIpPrefix(externalIp).build();
1052 ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1054 //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1055 ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1056 if (result.isSuccessful()) {
1057 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1058 GenerateVpnLabelOutput output = result.getResult();
1059 final Uint32 label = output.getLabel();
1061 int externalIpInDsFlag = 0;
1062 //Get IPMaps from the DB for the router ID
1063 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1064 for (IpMap dbIpMap : dbIpMaps) {
1065 String dbExternalIp = dbIpMap.getExternalIp();
1066 //Select the IPMap, whose external IP is the IP for which FIB is installed
1067 if (dbExternalIp.contains(externalIp)) {
1068 String dbInternalIp = dbIpMap.getInternalIp();
1069 IpMapKey dbIpMapKey = dbIpMap.key();
1070 LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1071 + "and externalIp {}", label, dbInternalIp, externalIp);
1072 IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1073 .setExternalIp(dbExternalIp).setLabel(label).build();
1074 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1075 naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1076 externalIpInDsFlag++;
1079 if (externalIpInDsFlag <= 0) {
1080 LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1081 + "Failed to update label {} for routerId {} in DS",
1082 externalIp, label, routerId);
1083 String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1084 + " found in DS for router %s", label, externalIp, routerId);
1085 return Futures.immediateFailedFuture(new Exception(errMsg));
1088 Uint32 l3vni = Uint32.ZERO;
1089 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1090 l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni);
1092 Routers extRouter = router != null ? router :
1093 NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1094 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
1095 externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1096 RouteOrigin.STATIC, dpnId);
1098 //Install custom FIB routes
1099 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1100 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1101 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1103 makeLFibTableEntry(dpnId, label, tableId, confTx);
1105 //Install custom FIB routes - FIB table.
1106 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1107 routerName, externalIp);
1108 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1109 //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1110 NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1112 String externalVpn = vpnName;
1113 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp, extRouter);
1114 if (extNwProvType == ProviderTypes.VLAN || extNwProvType == ProviderTypes.FLAT) {
1115 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1117 if (externalSubnet.isPresent()) {
1118 externalVpn = externalSubnetId.getValue();
1121 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1122 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1123 .setVpnName(externalVpn)
1124 .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1125 .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1126 .setInstruction(fibTableCustomInstructions).build();
1127 return fibService.createFibEntry(input);
1129 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1130 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1131 externalIp, vpnName, result.getErrors());
1132 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1134 }, MoreExecutors.directExecutor());
1136 Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1139 public void onFailure(@NonNull Throwable error) {
1140 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1144 public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
1145 if (result.isSuccessful()) {
1146 LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1149 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1150 + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1153 }, MoreExecutors.directExecutor());
1156 private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1157 String externalIp) {
1158 List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1159 Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1160 Uint32 externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1161 externalIp, router);
1162 int instructionIndex = 0;
1163 if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1164 Uint64 subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId.longValue());
1165 fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1166 MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1170 fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1171 return fibTableCustomInstructions;
1174 private void makeLFibTableEntry(Uint64 dpId, Uint32 serviceId, short tableId,
1175 TypedWriteTransaction<Configuration> confTx) {
1176 List<MatchInfo> matches = new ArrayList<>();
1177 matches.add(MatchEthernetType.MPLS_UNICAST);
1178 matches.add(new MatchMplsLabel(serviceId.longValue()));
1180 List<Instruction> instructions = new ArrayList<>();
1181 List<ActionInfo> actionsInfos = new ArrayList<>();
1182 //NAT is required for IPv4 only. Hence always etherType will be IPv4
1183 actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1184 Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1185 instructions.add(writeInstruction);
1186 instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1188 // Install the flow entry in L3_LFIB_TABLE
1189 String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1191 Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1193 COOKIE_VM_LFIB_TABLE, matches, instructions);
1195 mdsalManager.addFlow(confTx, dpId, flowEntity);
1197 LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1200 private void makeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId, Uint32 l3Vni,
1201 List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1202 ProviderTypes extNwProvType) {
1204 List<MatchInfo> mkMatches = new ArrayList<>();
1206 LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1207 dpnId, serviceId, customInstructions);
1209 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1210 mkMatches.add(new MatchTunnelId(Uint64.valueOf(l3Vni)));
1212 mkMatches.add(new MatchTunnelId(Uint64.valueOf(serviceId)));
1215 Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1216 getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""),
1217 NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1218 String.format("%s:%s", "TST Flow Entry ", serviceId), 0, 0,
1219 Uint64.valueOf(COOKIE_TUNNEL.toJava().add(BigInteger.valueOf(serviceId.longValue()))),
1220 mkMatches, customInstructions);
1222 mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1225 protected InstanceIdentifier<RouterIds> getRoutersIdentifier(Uint32 routerId) {
1226 return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1227 new RouterIdsKey(routerId)).build();
1230 private String getFlowRef(Uint64 dpnId, short tableId, Uint32 id, String ipAddress) {
1231 return NatConstants.SNAT_FLOWID_PREFIX + dpnId.toString() + NwConstants.FLOWID_SEPARATOR + tableId
1232 + NwConstants.FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1236 protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1237 LOG.trace("update : origRouter: {} updatedRouter: {}", original, update);
1238 String routerName = original.getRouterName();
1239 Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
1240 if (routerId == NatConstants.INVALID_ID) {
1241 LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1244 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1245 List<ListenableFuture<Void>> futures = new ArrayList<>();
1246 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1247 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1248 Uint32 bgpVpnId = NatConstants.INVALID_ID;
1249 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1250 if (bgpVpnUuid != null) {
1251 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1253 //BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1254 /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1255 * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1257 Uint64 dpnId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1258 boolean isPrimaryNaptSwitchNotSelected = (dpnId == null || dpnId.equals(Uint64
1259 .valueOf(BigInteger.ZERO)));
1260 Uuid networkId = original.getNetworkId();
1261 // Check if its update on SNAT flag
1262 boolean originalSNATEnabled = original.isEnableSnat();
1263 boolean updatedSNATEnabled = update.isEnableSnat();
1264 LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1265 + "as {} and {} with Elected Dpn {}(isPrimaryNaptSwitchNotSelected:{})",
1266 originalSNATEnabled, updatedSNATEnabled, dpnId, isPrimaryNaptSwitchNotSelected);
1267 // Cluster Reboot Case Handling
1268 // 1. DPN not elected during add event(due to none of the OVS connected)
1269 // 2. Update event called with changes of parameters(but enableSnat is not changed)
1270 // 3. First Elect dpnId and process other changes with valid dpnId
1271 if (originalSNATEnabled != updatedSNATEnabled || isPrimaryNaptSwitchNotSelected) {
1272 if (originalSNATEnabled && !updatedSNATEnabled) {
1273 if (isPrimaryNaptSwitchNotSelected) {
1274 LOG.info("No Action to be taken when SNAT is disabled "
1275 + "with no Napt Switch Election for Router {}", routerName);
1278 //SNAT disabled for the router
1279 Uuid networkUuid = original.getNetworkId();
1280 LOG.info("update : SNAT disabled for Router {}", routerName);
1281 Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1282 handleDisableSnat(original, networkUuid, externalIps, false, null,
1283 dpnId, routerId, removeFlowInvTx);
1284 } else if (updatedSNATEnabled) {
1285 LOG.info("update : SNAT enabled for Router {}", routerName);
1286 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid,
1287 true, writeFlowInvTx);
1288 if (isPrimaryNaptSwitchNotSelected) {
1289 dpnId = selectNewNAPTSwitch(routerName);
1290 if (dpnId != null && !dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1291 handleEnableSnat(update, routerId, dpnId, bgpVpnId, removeFlowInvTx);
1293 LOG.error("update : Failed to elect Napt Switch During update event"
1294 + " of router {}", routerName);
1298 LOG.info("update : no need to process external/subnet changes as it's will taken care"
1299 + "in handleDisableSnat/handleEnableSnat");
1302 if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1303 NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1304 NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1307 if (updatedSNATEnabled != originalSNATEnabled) {
1308 LOG.info("update : no need to process external/subnet changes as it's will taken care in "
1309 + "handleDisableSnat/handleEnableSnat");
1312 //Check if the Update is on External IPs
1313 LOG.debug("update : Checking if this is update on External IPs for router {}", routerName);
1314 List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1315 List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1317 //Check if the External IPs are removed during the update.
1318 Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1319 removedExternalIps.removeAll(updatedExternalIps);
1320 if (removedExternalIps.size() > 0) {
1321 LOG.debug("update : Start processing of the External IPs removal for router {}", routerName);
1322 vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1323 removedExternalIps, original.getExtGwMacAddress(),
1326 for (String removedExternalIp : removedExternalIps) {
1328 1) Remove the mappings in the IntExt IP model which has external IP.
1329 2) Remove the external IP in the ExternalCounter model.
1330 3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1331 least loaded external IP.
1332 Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1333 4) Increase the count of the allocated external IP by one.
1334 5) Advertise to the BGP if external IP is allocated for the first time for the router
1335 i.e. the route for the external IP is absent.
1336 6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1337 the removed external IPs and also from the model.
1338 7) Advertise to the BGP for removing the route for the removed external IPs.
1341 String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1342 String externalIp = externalIpParts[0];
1343 String externalIpPrefix = externalIpParts[1];
1344 String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1346 LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1347 + "entries for removed external IP {}", externalIpAddrStr);
1348 Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1349 String vpnName = "";
1350 if (vpnUuId != null) {
1351 vpnName = vpnUuId.getValue();
1353 clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1354 update.getExtGwMacAddress(), removeFlowInvTx);
1356 LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1357 //Get the internal IPs which are associated to the removed external IPs
1358 List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1359 List<String> removedInternalIps = new ArrayList<>();
1360 for (IpMap ipMap : ipMaps) {
1361 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1362 removedInternalIps.add(ipMap.getInternalIp());
1366 LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1367 for (String removedInternalIp : removedInternalIps) {
1368 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1369 + "router ID {} from the IntExtIP model",
1370 removedInternalIp, routerId);
1371 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1374 LOG.debug("update : Remove the count mapping of the external IP {} for the "
1375 + "router ID {} from the ExternalIpsCounter model.",
1376 externalIpAddrStr, routerId);
1377 naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1379 LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1380 + "whose external IPs were removed.");
1381 for (String removedInternalIp : removedInternalIps) {
1382 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1383 removedInternalIp, writeFlowInvTx);
1385 LOG.debug("update : Remove the NAPT translation entries from "
1386 + "Inbound and Outbound NAPT tables for the removed external IPs.");
1387 //Get the internalIP and internal Port which were associated to the removed external IP.
1388 Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1389 InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1390 .builder(IntextIpPortMap.class)
1391 .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1392 Optional<IpPortMapping> ipPortMapping;
1394 ipPortMapping = SingleTransactionDataBroker
1395 .syncReadOptional(dataBroker,
1396 LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1397 } catch (ReadFailedException e) {
1398 LOG.error("Failed to read ipPortMapping for router id {}", routerId, e);
1399 ipPortMapping = Optional.absent();
1402 if (ipPortMapping.isPresent()) {
1403 for (IntextIpProtocolType intextIpProtocolType :
1404 ipPortMapping.get().nonnullIntextIpProtocolType()) {
1405 ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1406 for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
1407 IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1408 if (ipPortExternal.getIpAddress().equals(externalIp)) {
1409 List<String> removedInternalIpPorts =
1410 protoTypesIntIpPortsMap.get(protoType);
1411 if (removedInternalIpPorts != null) {
1412 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1413 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1415 removedInternalIpPorts = new ArrayList<>();
1416 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1417 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1424 //Remove the IP port map from the intext-ip-port-map model, which were containing
1425 // the removed external IP.
1426 Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1427 protoTypesIntIpPortsMap.entrySet();
1428 Map<String, List<String>> internalIpPortMap = new HashMap<>();
1429 for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1430 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1431 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1432 for (String removedInternalIpPort : removedInternalIpPorts) {
1433 // Remove the IP port map from the intext-ip-port-map model,
1434 // which were containing the removed external IP
1435 naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1437 //Remove the IP port incomint packer map.
1438 naptPacketInHandler.removeIncomingPacketMap(
1439 routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1440 String[] removedInternalIpPortParts = removedInternalIpPort
1441 .split(NatConstants.COLON_SEPARATOR);
1442 if (removedInternalIpPortParts.length == 2) {
1443 String removedInternalIp = removedInternalIpPortParts[0];
1444 String removedInternalPort = removedInternalIpPortParts[1];
1445 List<String> removedInternalPortsList =
1446 internalIpPortMap.get(removedInternalPort);
1447 if (removedInternalPortsList != null) {
1448 removedInternalPortsList.add(removedInternalPort);
1449 internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1450 naptPacketInHandler.removeIncomingPacketMap(routerId
1451 + NatConstants.COLON_SEPARATOR + removedInternalIp
1452 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1453 //Remove the NAPT translation entries from Outbound NAPT table
1454 naptEventHandler.removeNatFlows(dpnId,
1455 NwConstants.OUTBOUND_NAPT_TABLE,
1456 routerId, removedInternalIp,
1457 Integer.parseInt(removedInternalPort),
1458 protocolType.getName());
1459 naptEventHandler.removeNatFlows(dpnId,
1460 NwConstants.INBOUND_NAPT_TABLE,
1461 routerId, removedInternalIp,
1462 Integer.parseInt(removedInternalPort),
1463 protocolType.getName());
1465 removedInternalPortsList = new ArrayList<>();
1466 removedInternalPortsList.add(removedInternalPort);
1467 internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1468 naptPacketInHandler.removeIncomingPacketMap(routerId
1469 + NatConstants.COLON_SEPARATOR + removedInternalIp
1470 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1471 //Remove the NAPT translation entries from Outbound NAPT table
1472 naptEventHandler.removeNatFlows(dpnId,
1473 NwConstants.OUTBOUND_NAPT_TABLE,
1474 routerId, removedInternalIp,
1475 Integer.parseInt(removedInternalPort),
1476 protocolType.getName());
1477 naptEventHandler.removeNatFlows(dpnId,
1478 NwConstants.INBOUND_NAPT_TABLE, routerId, removedInternalIp,
1479 Integer.parseInt(removedInternalPort),
1480 protocolType.getName());
1486 // Delete the entry from SnatIntIpPortMap DS
1487 Set<String> internalIps = internalIpPortMap.keySet();
1488 for (String internalIp : internalIps) {
1489 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1490 + "model SnatIntIpPortMap", internalIp);
1491 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1494 naptManager.removeNaptPortPool(externalIp);
1497 "update : End processing of the External IPs removal for router {}", routerName);
1500 //Check if the External IPs are added during the update.
1501 Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1502 addedExternalIps.removeAll(originalExternalIps);
1503 if (addedExternalIps.size() != 0) {
1504 LOG.debug("update : Start processing of the External IPs addition for router {}",
1506 vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1507 update.getExtGwMacAddress(), dpnId,
1508 update.getNetworkId());
1510 for (String addedExternalIp : addedExternalIps) {
1512 1) Do nothing in the IntExtIp model.
1513 2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1515 String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1516 String externalIp = externalIpParts[0];
1517 String externalIpPrefix = externalIpParts[1];
1518 String externalpStr = externalIp + "/" + externalIpPrefix;
1519 LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1520 + "router ID {} in the ExternalIpsCounter model.",
1521 externalpStr, routerId);
1522 naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1523 subnetRegisterMapping(update, routerId);
1524 LOG.info("update : Installing fib flow fo newly added Ips");
1525 handleSnatReverseTraffic(writeFlowInvTx, dpnId, update, routerId, routerName, externalpStr);
1528 "update : End processing of the External IPs addition during the update operation");
1531 //Check if its Update on subnets
1532 LOG.debug("update : Checking if this is update on subnets for router {}", routerName);
1533 List<Uuid> originalSubnetIds = original.getSubnetIds();
1534 List<Uuid> updatedSubnetIds = update.getSubnetIds();
1535 Set<Uuid> addedSubnetIds =
1536 updatedSubnetIds != null ? new HashSet<>(updatedSubnetIds) : new HashSet<>();
1537 if (originalSubnetIds != null) {
1538 addedSubnetIds.removeAll(originalSubnetIds);
1541 //Check if the Subnet IDs are added during the update.
1542 if (addedSubnetIds.size() != 0) {
1544 "update : Start processing of the Subnet IDs addition for router {}", routerName);
1545 for (Uuid addedSubnetId : addedSubnetIds) {
1547 1) Select the least loaded external IP for the subnet and store the mapping of the
1548 subnet IP and the external IP in the IntExtIp model.
1549 2) Increase the count of the selected external IP by one.
1550 3) Advertise to the BGP if external IP is allocated for the first time for the
1551 router i.e. the route for the external IP is absent.
1553 String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1554 if (subnetIp != null) {
1555 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1559 LOG.debug("update : End processing of the Subnet IDs addition for router {}", routerName);
1562 //Check if the Subnet IDs are removed during the update.
1563 Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1564 removedSubnetIds.removeAll(updatedSubnetIds);
1565 if (removedSubnetIds.size() != 0) {
1567 "update : Start processing of the Subnet IDs removal for router {}", routerName);
1568 for (Uuid removedSubnetId : removedSubnetIds) {
1569 String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1570 if (subnetAddr != null) {
1572 1) Remove the subnet IP and the external IP in the IntExtIp map
1573 2) Decrease the count of the coresponding external IP by one.
1574 3) Advertise to the BGP for removing the routes of the corresponding external
1575 IP if its not allocated to any other internal IP.
1578 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1579 subnetAddr[0] + "/" + subnetAddr[1]);
1580 if (externalIp == null) {
1581 LOG.error("update : No mapping found for router ID {} and internal IP {}",
1582 routerId, subnetAddr[0]);
1586 naptManager.updateCounter(routerId, externalIp, false);
1587 // Traverse entire model of external-ip counter whether external ip is not
1588 // used by any other internal ip in any router
1589 if (!isExternalIpAllocated(externalIp)) {
1590 LOG.debug("update : external ip is not allocated to any other "
1591 + "internal IP so proceeding to remove routes");
1592 clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1593 Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1595 LOG.debug("update : Successfully removed fib entries in switch {} for "
1596 + "router {} with networkId {} and externalIp {}",
1597 dpnId, routerId, networkId, externalIp);
1600 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1601 + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1602 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1605 LOG.debug("update : End processing of the Subnet IDs removal for router {}", routerName);
1610 }, NatConstants.NAT_DJC_MAX_RETRIES);
1613 private boolean isExternalIpAllocated(String externalIp) {
1614 InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1615 Optional<ExternalIpsCounter> externalCountersData;
1617 externalCountersData = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1618 LogicalDatastoreType.OPERATIONAL, id);
1619 } catch (ReadFailedException e) {
1620 LOG.error("Failed to read external counters data for ExternalIp {}", externalIp, e);
1621 externalCountersData = Optional.absent();
1623 if (externalCountersData.isPresent()) {
1624 ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1625 for (ExternalCounters ext : externalIpsCounters.nonnullExternalCounters()) {
1626 for (ExternalIpCounter externalIpCount : ext.nonnullExternalIpCounter()) {
1627 if (externalIpCount.getExternalIp().equals(externalIp)) {
1628 if (externalIpCount.getCounter().toJava() != 0) {
1639 private void allocateExternalIp(Uint64 dpnId, Routers router, Uint32 routerId, String routerName,
1640 Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1641 String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1643 InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1644 if (address instanceof Inet6Address) {
1645 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1648 } catch (UnknownHostException e) {
1649 LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1652 String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1653 if (leastLoadedExtIpAddr != null) {
1654 String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1655 String leastLoadedExtIp = externalIpParts[0];
1656 String leastLoadedExtIpPrefix = externalIpParts[1];
1657 IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1658 subnetIp = subnetIpParts[0];
1659 String subnetIpPrefix = subnetIpParts[1];
1660 IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1661 LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1662 + "IP {} and prefix {} -> external IP {} and prefix {}",
1663 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1664 naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1667 // Check if external IP is already assigned a route. (i.e. External IP is previously
1668 // allocated to any of the subnets)
1669 // If external IP is already assigned a route, (, do not re-advertise to the BGP
1670 String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1671 Uint32 label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1672 if (label != null) {
1674 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1675 IpMapKey ipMapKey = new IpMapKey(internalIp);
1676 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1677 label, internalIp, leastLoadedExtIpAddrStr);
1678 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1679 .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1680 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1681 naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1685 // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1686 // for the first time and hence not having a route.
1687 //Get the VPN Name using the network ID
1688 final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1689 if (vpnName != null) {
1690 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1691 if (dpnId == null || dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1692 LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1693 + "added after gateway-set");
1694 dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1695 if (dpnId == null || dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1696 LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1700 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1701 leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1708 protected Uint32 checkExternalIpLabel(Uint32 routerId, String externalIp) {
1709 List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1710 for (IpMap ipMap : ipMaps) {
1711 if (ipMap.getExternalIp().equals(externalIp)) {
1712 if (ipMap.getLabel() != null) {
1713 return ipMap.getLabel();
1717 LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1722 protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1723 LOG.trace("remove : Router delete method");
1725 ROUTER DELETE SCENARIO
1726 1) Get the router ID from the event.
1727 2) Build the cookie information from the router ID.
1728 3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1729 4) Build the flow with the cookie value.
1730 5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1731 6) Remove the flows from the other switches which points to the primary and secondary
1732 switches for the flows related the router ID.
1733 7) Get the list of external IP address maintained for the router ID.
1734 8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1735 9) Withdraw the corresponding routes from the BGP.
1738 if (identifier == null || router == null) {
1739 LOG.error("remove : returning without processing since routers is null");
1743 String routerName = router.getRouterName();
1744 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1745 () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1746 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1748 Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
1749 if (routerId == NatConstants.INVALID_ID) {
1750 LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1754 Uint32 bgpVpnId = NatConstants.INVALID_ID;
1755 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1756 if (bgpVpnUuid != null) {
1757 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1759 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1761 Uuid networkUuid = router.getNetworkId();
1763 Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1764 if (primarySwitchId == null || primarySwitchId.equals(Uint64.ZERO)) {
1765 // No NAPT switch for external router, probably because the router is not attached to
1767 // internal networks
1769 "No NAPT switch for router {}, check if router is attached to any internal "
1774 Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1775 handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1778 if (NatUtil.releaseId(idManager, NatConstants.ODL_VNI_POOL_NAME, routerName)
1779 == NatConstants.INVALID_ID) {
1780 LOG.error("remove: Unable to release VNI for router - {}", routerName);
1782 })), NatConstants.NAT_DJC_MAX_RETRIES);
1785 public void handleDisableSnat(Routers router, Uuid networkUuid, @NonNull Collection<String> externalIps,
1786 boolean routerFlag, @Nullable String vpnName, Uint64 naptSwitchDpnId,
1787 Uint32 routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1788 LOG.info("handleDisableSnat : Entry");
1789 String routerName = router.getRouterName();
1792 removeNaptSwitch(routerName);
1794 updateNaptSwitch(routerName, Uint64.valueOf(BigInteger.ZERO));
1797 LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1798 naptManager.removeExternalCounter(routerId);
1800 LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1801 if (naptSwitchDpnId == null || naptSwitchDpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1802 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1803 + "router ID {} from RouterNaptSwitch model", routerId);
1806 ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1808 if (extNwProvType == null) {
1809 LOG.error("handleDisableSnat : External Network Provider Type missing");
1812 Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1813 removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1814 externalSubnetList, removeFlowInvTx, extNwProvType);
1815 removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1816 String externalSubnetVpn = null;
1817 for (Uuid externalSubnetId : externalSubnetList) {
1818 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1819 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1820 if (externalSubnet.isPresent()) {
1821 externalSubnetVpn = externalSubnetId.getValue();
1822 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1823 router.getExtGwMacAddress(), removeFlowInvTx);
1826 if (externalSubnetVpn == null) {
1827 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1828 router.getExtGwMacAddress(), removeFlowInvTx);
1830 // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1831 // for the router ID.
1832 LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1833 + "router ID {} in the DS", routerId);
1834 naptManager.removeMapping(routerId);
1835 } catch (InterruptedException | ExecutionException e) {
1836 LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, e);
1838 LOG.info("handleDisableSnat : Exit");
1841 // TODO Clean up the exception handling
1842 @SuppressWarnings("checkstyle:IllegalCatch")
1843 public void handleDisableSnatInternetVpn(String routerName, Uint32 routerId, Uuid networkUuid,
1844 @NonNull Collection<String> externalIps,
1845 String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1846 LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1847 + "with internet vpn {}", routerName, vpnId);
1849 Uint64 naptSwitchDpnId = null;
1850 InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1851 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1852 Optional<RouterToNaptSwitch> rtrToNapt;
1854 rtrToNapt = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1855 LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1856 } catch (ReadFailedException e) {
1857 LOG.error("Failed to read NAPT switch for router {}", routerName, e);
1858 rtrToNapt = Optional.absent();
1860 if (rtrToNapt.isPresent()) {
1861 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1863 LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1865 removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1868 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1869 if (extGwMacAddress != null) {
1870 LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1871 + "External Router ID {}", extGwMacAddress, routerId);
1873 LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1874 + "External Router ID {}", routerId);
1877 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1879 } catch (Exception ex) {
1880 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1881 + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1883 if (NatUtil.releaseId(idManager, NatConstants.ODL_VNI_POOL_NAME, vpnId) == NatConstants.INVALID_ID) {
1884 LOG.error("handleDisableSnatInternetVpn : Unable to release VNI for vpnId {} ", vpnId);
1886 } catch (InterruptedException | ExecutionException e) {
1887 LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1888 + "with internet vpn {}", routerName, vpnId, e);
1890 LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1894 // TODO Clean up the exception handling
1895 @SuppressWarnings("checkstyle:IllegalCatch")
1896 public void updateNaptSwitch(String routerName, Uint64 naptSwitchId) {
1897 RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1898 .setPrimarySwitchId(naptSwitchId).build();
1900 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1901 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1902 } catch (Exception ex) {
1903 LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1904 naptSwitchId, routerName);
1906 LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1907 naptSwitchId, routerName);
1910 protected void removeNaptSwitch(String routerName) {
1911 // Remove router and switch from model
1912 InstanceIdentifier<RouterToNaptSwitch> id =
1913 InstanceIdentifier.builder(NaptSwitches.class)
1914 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1915 LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1916 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1917 //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1918 NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1921 public void removeNaptFlowsFromActiveSwitch(Uint32 routerId, String routerName,
1922 Uint64 dpnId, Uuid networkId, String vpnName,
1923 @NonNull Collection<String> externalIps,
1924 Collection<Uuid> externalSubnetList,
1925 TypedReadWriteTransaction<Configuration> confTx,
1926 ProviderTypes extNwProvType)
1927 throws InterruptedException, ExecutionException {
1929 LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1930 Uint64 cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1932 //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1933 // traffic which comes from the VMs of the NAPT switches)
1934 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1935 FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1938 "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1939 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1940 mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1942 //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1943 // traffic which comes from the VMs of the non NAPT switches)
1944 Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
1945 elanManager, idManager, routerId, routerName);
1946 String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, Uint32.valueOf(tunnelId.longValue()));
1947 FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1949 "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1950 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1951 mdsalManager.removeFlow(confTx, tsNatFlowEntity);
1953 //Remove the flow table 25->44 from NAPT Switch
1954 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1955 NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
1958 //Remove the Outbound flow entry which forwards the packet to FIB Table
1960 "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1961 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1963 String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1964 NwConstants.IP_PROT_TCP);
1965 FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1966 outboundTcpNatFlowRef);
1967 mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
1969 String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1970 NwConstants.IP_PROT_UDP);
1971 FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1972 outboundUdpNatFlowRef);
1973 mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
1975 String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1976 NwConstants.IP_PROT_ICMP);
1977 FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1979 mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
1980 boolean lastRouterOnExternalNetwork =
1981 !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker, networkId, routerName, dpnId);
1982 if (lastRouterOnExternalNetwork) {
1983 removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
1985 //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1986 // External Subnet Vpn Id.
1987 for (Uuid externalSubnetId : externalSubnetList) {
1988 Uint32 subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1989 if (subnetVpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
1990 dataBroker, externalSubnetId, routerName, dpnId)) {
1991 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1992 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1993 natPfibSubnetFlowRef);
1994 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1995 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1996 + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1997 subnetVpnId, dpnId);
2001 //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
2002 String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
2003 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
2006 "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
2007 + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
2008 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
2010 if (lastRouterOnExternalNetwork) {
2011 // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
2012 // - This does not work since ext-routers is deleted already - no network info
2013 //Get the VPN ID from the ExternalNetworks model
2014 Uint32 vpnId = NatConstants.INVALID_ID;
2015 if (vpnName == null || vpnName.isEmpty()) {
2016 // ie called from router delete cases
2017 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2018 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
2019 if (vpnUuid != null) {
2020 vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2021 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external network {} router delete "
2022 + "or disableSNAT scenario", vpnId, networkId);
2025 // ie called from disassociate vpn case
2026 LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
2028 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2029 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}",
2033 if (vpnId != NatConstants.INVALID_ID) {
2034 //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2035 String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2036 FlowEntity natPfibVpnFlowEntity =
2037 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2038 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the "
2039 + "DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2040 mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2044 //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2045 IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2046 if (ipPortMapping == null) {
2047 LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2051 for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2052 String protocol = intextIpProtocolType.getProtocol().name();
2053 for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2054 String ipPortInternal = ipPortMap.getIpPortInternal();
2055 String[] ipPortParts = ipPortInternal.split(":");
2056 if (ipPortParts.length != 2) {
2057 LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2060 String internalIp = ipPortParts[0];
2061 String internalPort = ipPortParts[1];
2063 //Build the flow for the outbound NAPT table
2064 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2065 + NatConstants.COLON_SEPARATOR + internalPort);
2066 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2067 String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2068 FlowEntity outboundNaptFlowEntity =
2069 NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2071 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2072 + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2073 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2075 //Build the flow for the inbound NAPT table
2076 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2077 String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2078 FlowEntity inboundNaptFlowEntity =
2079 NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2081 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2082 + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2083 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2088 protected void removeNaptFibExternalOutputFlows(Uint32 routerId, Uint64 dpnId, Uuid networkId,
2089 @NonNull Collection<String> externalIps,
2090 TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2091 throws ExecutionException, InterruptedException {
2092 Uint32 extVpnId = NatConstants.INVALID_ID;
2093 if (networkId != null) {
2094 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2095 if (vpnUuid != null) {
2096 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2098 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2101 LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2102 extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2104 if (extVpnId == NatConstants.INVALID_ID) {
2105 LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2106 extVpnId = routerId;
2108 for (String ip : externalIps) {
2109 String extIp = removeMaskFromIp(ip);
2110 String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2111 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2112 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2113 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2114 FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2115 mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2119 private String removeMaskFromIp(String ip) {
2120 if (ip != null && !ip.trim().isEmpty()) {
2121 return ip.split("/")[0];
2126 public void removeNaptFlowsFromActiveSwitchInternetVpn(Uint32 routerId, String routerName,
2127 Uint64 dpnId, Uuid networkId, String vpnName,
2128 TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2129 throws ExecutionException, InterruptedException {
2130 LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2131 Uint64 cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2133 //Remove the NAPT PFIB TABLE entry
2134 Uint32 vpnId = NatConstants.INVALID_ID;
2135 if (vpnName != null) {
2136 // ie called from disassociate vpn case
2137 LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2138 + "with vpnName {}", vpnName);
2139 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2140 LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2144 if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2145 networkId, routerName, dpnId)) {
2146 //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2147 String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2148 FlowEntity natPfibVpnFlowEntity =
2149 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2150 LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2151 + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2152 mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2154 // Remove IP-PORT active NAPT entries and release port from IdManager
2155 // For the router ID get the internal IP , internal port and the corresponding
2156 // external IP and external Port.
2157 IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2158 if (ipPortMapping == null) {
2159 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2162 for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2163 String protocol = intextIpProtocolType.getProtocol().name();
2164 for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2165 String ipPortInternal = ipPortMap.getIpPortInternal();
2166 String[] ipPortParts = ipPortInternal.split(":");
2167 if (ipPortParts.length != 2) {
2168 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2172 String internalIp = ipPortParts[0];
2173 String internalPort = ipPortParts[1];
2175 //Build the flow for the outbound NAPT table
2176 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2177 + NatConstants.COLON_SEPARATOR + internalPort);
2178 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2179 String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2180 FlowEntity outboundNaptFlowEntity =
2181 NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2183 LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2184 + "active switch with the DPN ID {} and router ID {}",
2185 NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2186 mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2188 IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2189 final String externalIp = ipPortExternal.getIpAddress();
2191 //Build the flow for the inbound NAPT table
2192 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2193 String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2194 FlowEntity inboundNaptFlowEntity =
2195 NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2197 LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2198 + "active active switch with the DPN ID {} and router ID {}",
2199 NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2200 mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2202 // Finally release port from idmanager
2203 String internalIpPort = internalIp + ":" + internalPort;
2204 naptManager.removePortFromPool(internalIpPort, externalIp);
2206 //Remove sessions from models
2207 naptManager.removeIpPortMappingForRouterID(routerId);
2208 naptManager.removeIntIpPortMappingForRouterID(routerId);
2212 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2216 public void removeFlowsFromNonActiveSwitches(Uint32 routerId, String routerName,
2217 Uint64 naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2218 throws ExecutionException, InterruptedException {
2219 LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2221 // Remove the flows from the other switches which points to the primary and secondary switches
2222 // for the flows related the router ID.
2223 List<Uint64> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2224 if (allSwitchList.isEmpty()) {
2225 LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2228 for (Uint64 dpnId : allSwitchList) {
2229 if (!naptSwitchDpnId.equals(dpnId)) {
2230 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2232 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2233 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2234 FlowEntity preSnatFlowEntity =
2235 NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2237 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2238 + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2239 mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2241 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2242 Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
2243 NatUtil.getGroupIdKey(routerName));
2244 if (groupId != NatConstants.INVALID_ID) {
2246 "removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2247 + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2248 mdsalManager.removeGroup(removeFlowInvTx, dpnId, groupId.longValue());
2250 LOG.error("removeFlowsFromNonActiveSwitches: Unable to obtained groupID for router:{}", routerName);
2256 public void clrRtsFromBgpAndDelFibTs(final Uint64 dpnId, Uint32 routerId, @Nullable Uuid networkUuid,
2257 @NonNull Collection<String> externalIps, @Nullable String vpnName,
2258 String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx)
2259 throws ExecutionException, InterruptedException {
2260 //Withdraw the corresponding routes from the BGP.
2261 //Get the network ID using the router ID.
2262 LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2263 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2264 if (networkUuid == null) {
2265 LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2269 if (externalIps.isEmpty()) {
2270 LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2274 if (vpnName == null) {
2275 //Get the VPN Name using the network ID
2276 vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2277 if (vpnName == null) {
2278 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2279 networkUuid, routerId);
2283 LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2285 //Remove custom FIB routes
2286 //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2287 for (String extIp : externalIps) {
2288 clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2292 protected void clrRtsFromBgpAndDelFibTs(final Uint64 dpnId, Uint32 routerId, String extIp, final String vpnName,
2293 final Uuid networkUuid, String extGwMacAddress,
2294 TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2295 throws ExecutionException, InterruptedException {
2296 clearBgpRoutes(extIp, vpnName);
2297 delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2301 protected void delFibTsAndReverseTraffic(final Uint64 dpnId, String routerName, Uint32 routerId, String extIp,
2302 String vpnName, Uuid extNetworkId, Uint32 tempLabel,
2303 String gwMacAddress, boolean switchOver,
2304 TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2305 throws ExecutionException, InterruptedException {
2306 LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2307 //String routerName = NatUtil.getRouterName(dataBroker,routerId);
2308 if (routerName == null) {
2309 LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2312 ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2313 if (extNwProvType == null) {
2314 LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2317 /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2318 * external network provided type is VxLAN
2320 if (extNwProvType == ProviderTypes.VXLAN) {
2321 evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2325 if (tempLabel.longValue() < 0) {
2326 LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2330 final Uint32 label = tempLabel;
2331 final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2332 if (extNwProvType == ProviderTypes.FLAT || extNwProvType == ProviderTypes.VLAN) {
2333 LOG.debug("delFibTsAndReverseTraffic : Using extSubnetId as vpnName for FLAT/VLAN use-cases");
2334 Routers extRouter = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
2335 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
2338 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
2341 if (externalSubnet.isPresent()) {
2342 vpnName = externalSubnetId.getValue();
2345 final String externalVpn = vpnName;
2346 RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(externalVpn)
2347 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2348 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2349 ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2351 removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2352 removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2353 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2354 //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2355 NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2358 ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2359 Futures.transformAsync(future, result -> {
2361 if (result.isSuccessful()) {
2362 NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2363 RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2364 .setVpnName(externalVpn).setIpPrefix(externalIp).build();
2365 return vpnService.removeVpnLabel(labelInput);
2368 String.format("RPC call to remove custom FIB entries on dpn %s for "
2369 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2371 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2373 }, MoreExecutors.directExecutor());
2375 Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2378 public void onFailure(@NonNull Throwable error) {
2379 LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2380 + "got external ip {}", label, extIp, error);
2384 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2385 if (result.isSuccessful()) {
2386 LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2387 + "from VPN {}", externalIp, externalVpn);
2389 LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2390 + " from VPN {}, {}", externalIp, externalVpn, result.getErrors());
2393 }, MoreExecutors.directExecutor());
2395 LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2396 + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2400 private void delFibTsAndReverseTraffic(final Uint64 dpnId, Uint32 routerId, String extIp, final String vpnName,
2401 final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2402 TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2403 throws ExecutionException, InterruptedException {
2404 LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2405 String routerName = NatUtil.getRouterName(dataBroker,routerId);
2406 if (routerName == null) {
2407 LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2410 //Get the external network provider type from networkId
2411 ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2412 if (extNwProvType == null) {
2413 LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2417 /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2418 * external network provided type is VxLAN
2420 if (extNwProvType == ProviderTypes.VXLAN) {
2421 evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2424 //Get IPMaps from the DB for the router ID
2425 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2426 if (dbIpMaps.isEmpty()) {
2427 LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2431 Uint32 tempLabel = NatConstants.INVALID_ID;
2432 for (IpMap dbIpMap : dbIpMaps) {
2433 String dbExternalIp = dbIpMap.getExternalIp();
2434 LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2435 //Select the IPMap, whose external IP is the IP for which FIB is installed
2436 if (extIp.equals(dbExternalIp)) {
2437 tempLabel = dbIpMap.getLabel();
2438 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2439 tempLabel, dbExternalIp, routerId);
2443 if (tempLabel == NatConstants.INVALID_ID) {
2444 LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2449 final Uint32 label = tempLabel;
2450 final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2451 RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2452 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2453 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2454 ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2456 removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2457 removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2458 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2459 //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2460 NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2463 ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2464 Futures.transformAsync(future, result -> {
2466 if (result.isSuccessful()) {
2467 RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2468 .setVpnName(vpnName).setIpPrefix(externalIp).build();
2469 return vpnService.removeVpnLabel(labelInput);
2472 String.format("RPC call to remove custom FIB entries on dpn %s for "
2473 + "prefix %s Failed - %s",
2474 dpnId, externalIp, result.getErrors());
2476 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2478 }, MoreExecutors.directExecutor());
2480 Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2483 public void onFailure(@NonNull Throwable error) {
2484 LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2488 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2489 if (result.isSuccessful()) {
2490 LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2491 + "from VPN {}", externalIp, vpnName);
2493 LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2494 + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2497 }, MoreExecutors.directExecutor());
2499 LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2500 + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2504 protected void clearFibTsAndReverseTraffic(final Uint64 dpnId, Uint32 routerId, Uuid networkUuid,
2505 List<String> externalIps, @Nullable String vpnName, String extGwMacAddress,
2506 TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2507 //Withdraw the corresponding routes from the BGP.
2508 //Get the network ID using the router ID.
2509 LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2510 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2511 if (networkUuid == null) {
2512 LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2516 if (externalIps == null || externalIps.isEmpty()) {
2517 LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2521 if (vpnName == null) {
2522 //Get the VPN Name using the network ID
2523 vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2524 if (vpnName == null) {
2525 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2526 networkUuid, routerId);
2530 LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2532 //Remove custom FIB routes
2533 //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2534 for (String extIp : externalIps) {
2535 delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2540 protected void clearBgpRoutes(String externalIp, final String vpnName) {
2541 //Inform BGP about the route removal
2542 LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2543 String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2544 NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName);
2547 private void removeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId,
2548 TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2549 LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2550 mdsalManager.removeFlow(writeFlowInvTx, dpnId,
2551 getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), NwConstants.INTERNAL_TUNNEL_TABLE);
2552 LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2555 private void removeLFibTableEntry(Uint64 dpnId, Uint32 serviceId,
2556 TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2557 String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2558 LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2559 mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowRef, NwConstants.L3_LFIB_TABLE);
2560 LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2564 * router association to vpn.
2566 * @param routerName - Name of router
2567 * @param routerId - router id
2568 * @param bgpVpnName BGP VPN name
2570 public void changeLocalVpnIdToBgpVpnId(String routerName, Uint32 routerId, String bgpVpnName,
2571 TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2572 LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2573 if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2574 Uint32 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2576 LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2578 if (bgpVpnId != NatConstants.INVALID_ID) {
2579 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2580 + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2581 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2582 .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2583 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2584 getRoutersIdentifier(bgpVpnId), rtrs);
2586 // Get the allocated Primary NAPT Switch for this router
2587 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2588 routerId, bgpVpnId);
2589 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2592 Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2593 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2600 * router disassociation from vpn.
2602 * @param routerName - Name of router
2603 * @param routerId - router id
2604 * @param bgpVpnName BGP VPN name
2606 public void changeBgpVpnIdToLocalVpnId(String routerName, Uint32 routerId, String bgpVpnName,
2607 TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2608 LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2609 if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2610 Uint32 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2611 LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2613 // Get the allocated Primary NAPT Switch for this router
2614 LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2616 LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2617 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2620 Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2621 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2622 writeFlowInvTx, extNwProvType);
2626 boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2627 InstanceIdentifier<Routers> routerInstanceIndentifier =
2628 InstanceIdentifier.builder(ExtRouters.class)
2629 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2631 Optional<Routers> routerData = SingleTransactionDataBroker
2632 .syncReadOptional(dataBroker,
2633 LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2634 return routerData.isPresent() && routerData.get().isEnableSnat();
2635 } catch (ReadFailedException e) {
2636 LOG.error("Failed to read data for router id {}", routerUuid, e);
2641 public void installFlowsWithUpdatedVpnId(Uint64 primarySwitchId, String routerName, Uint32 bgpVpnId,
2642 Uint32 routerId, boolean isSnatCfgd,
2643 TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2645 Uint32 changedVpnId = bgpVpnId;
2646 String idType = "BGP VPN";
2647 if (bgpVpnId == NatConstants.INVALID_ID) {
2648 changedVpnId = routerId;
2652 List<Uint64> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2653 if (switches.isEmpty()) {
2654 LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2657 for (Uint64 dpnId : switches) {
2658 // Update the BGP VPN ID in the SNAT miss entry to group
2659 if (!dpnId.equals(primarySwitchId)) {
2660 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2661 List<BucketInfo> bucketInfoForNonNaptSwitches =
2662 getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2663 Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
2664 NatUtil.getGroupIdKey(routerName));
2665 if (groupId != NatConstants.INVALID_ID) {
2667 installGroup(dpnId, routerName, groupId, bucketInfoForNonNaptSwitches);
2670 "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2671 + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2672 FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName,
2673 groupId, changedVpnId);
2674 mdsalManager.addFlow(confTx, flowEntity);
2676 LOG.error("installFlowsWithUpdatedVpnId: Unable to get groupId for router:{}", routerName);
2680 "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2681 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2682 FlowEntity flowEntity =
2683 buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2684 mdsalManager.addFlow(confTx, flowEntity);
2687 "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2688 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2689 idType, changedVpnId, primarySwitchId);
2690 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2691 changedVpnId, confTx, extNwProvType);
2694 "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2695 + "which punts the packet to the controller in the Primary switch {}",
2696 idType, changedVpnId, primarySwitchId);
2697 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2700 "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2701 + " outgoing packet to FIB Table in the Primary switch {}",
2702 idType, changedVpnId, primarySwitchId);
2703 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2706 "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2707 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2708 + " {}", idType, changedVpnId, primarySwitchId);
2709 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2711 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2713 Uint32 vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2714 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2715 if (vpnId != NatConstants.INVALID_ID) {
2716 installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2722 public void updateNaptFlowsWithVpnId(Uint64 dpnId, String routerName, Uint32 routerId, Uint32 bgpVpnId) {
2723 //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2724 IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2725 if (ipPortMapping == null) {
2726 LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2729 // Get the External Gateway MAC Address
2730 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2731 if (extGwMacAddress != null) {
2732 LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2733 extGwMacAddress, routerId);
2735 LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2739 for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2740 for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2741 String ipPortInternal = ipPortMap.getIpPortInternal();
2742 String[] ipPortParts = ipPortInternal.split(":");
2743 if (ipPortParts.length != 2) {
2744 LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2747 String internalIp = ipPortParts[0];
2748 String internalPort = ipPortParts[1];
2749 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2750 internalIp, internalPort);
2751 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2752 NAPTEntryEvent.Protocol protocol;
2753 switch (protocolTypes) {
2755 protocol = NAPTEntryEvent.Protocol.TCP;
2758 protocol = NAPTEntryEvent.Protocol.UDP;
2761 protocol = NAPTEntryEvent.Protocol.TCP;
2763 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2764 SessionAddress externalAddress =
2765 naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2766 Uint32 internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2767 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2768 routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2769 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2770 routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2775 public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(Uint64 dpId, String routerName, Uint32 groupId,
2776 Uint32 changedVpnId) {
2778 LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2779 + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2780 List<MatchInfo> matches = new ArrayList<>();
2781 matches.add(MatchEthernetType.IPV4);
2782 matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2783 MetaDataUtil.METADATA_MASK_VRFID));
2785 List<ActionInfo> actionsInfo = new ArrayList<>();
2786 Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
2787 elanManager, idManager, changedVpnId, routerName);
2788 actionsInfo.add(new ActionSetFieldTunnelId(tunnelId));
2789 LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2791 actionsInfo.add(new ActionGroup(groupId.longValue()));
2792 List<InstructionInfo> instructions = new ArrayList<>();
2793 instructions.add(new InstructionApplyActions(actionsInfo));
2794 String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2795 FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2796 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2797 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2799 LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2803 public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(Uint64 dpId, String routerName,
2804 Uint32 changedVpnId) {
2806 LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2807 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2808 List<MatchInfo> matches = new ArrayList<>();
2809 matches.add(MatchEthernetType.IPV4);
2810 matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2811 MetaDataUtil.METADATA_MASK_VRFID));
2813 List<InstructionInfo> instructions = new ArrayList<>();
2814 instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2816 String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2817 FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2818 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2819 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2821 LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2825 // TODO : Replace this with ITM Rpc once its available with full functionality
2826 protected void installTerminatingServiceTblEntryWithUpdatedVpnId(Uint64 dpnId, String routerName,
2827 Uint32 routerId, Uint32 changedVpnId,
2828 TypedWriteTransaction<Configuration> confTx,
2829 ProviderTypes extNwProvType) {
2831 LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2832 + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2833 FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2835 mdsalManager.addFlow(confTx, flowEntity);
2838 private FlowEntity buildTsFlowEntityWithUpdatedVpnId(Uint64 dpId, String routerName,
2839 Uint32 routerIdLongVal, Uint32 changedVpnId,
2840 ProviderTypes extNwProvType) {
2841 LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2842 dpId, routerName, changedVpnId);
2843 List<MatchInfo> matches = new ArrayList<>();
2844 matches.add(MatchEthernetType.IPV4);
2846 Uint64 tunnelId = Uint64.valueOf(changedVpnId);
2847 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2848 tunnelId = natOverVxlanUtil.getRouterVni(routerName, changedVpnId);
2850 matches.add(new MatchTunnelId(tunnelId));
2852 List<InstructionInfo> instructions = new ArrayList<>();
2853 instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2854 MetaDataUtil.METADATA_MASK_VRFID));
2855 instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2856 Uint32 routerId = routerIdLongVal;
2857 String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId);
2858 FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2859 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2860 NwConstants.COOKIE_TS_TABLE, matches, instructions);
2864 public void createOutboundTblEntryWithBgpVpn(Uint64 dpnId, Uint32 routerId, Uint32 changedVpnId,
2865 TypedWriteTransaction<Configuration> writeFlowInvTx) {
2866 LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2867 dpnId, routerId, changedVpnId);
2868 FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2869 NwConstants.IP_PROT_TCP);
2870 LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2871 mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2873 FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2874 NwConstants.IP_PROT_UDP);
2875 LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2876 mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2878 FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2879 LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2880 mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2883 protected FlowEntity buildOutboundFlowEntityWithBgpVpn(Uint64 dpId, Uint32 routerId,
2884 Uint32 changedVpnId, int protocol) {
2885 LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2886 dpId, routerId, changedVpnId);
2887 Uint64 cookie = getCookieOutboundFlow(routerId);
2888 List<MatchInfo> matches = new ArrayList<>();
2889 matches.add(MatchEthernetType.IPV4);
2890 matches.add(new MatchIpProtocol((short)protocol));
2891 matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2892 MetaDataUtil.METADATA_MASK_VRFID));
2894 List<InstructionInfo> instructions = new ArrayList<>();
2895 List<ActionInfo> actionsInfos = new ArrayList<>();
2896 actionsInfos.add(new ActionPuntToController());
2897 if (snatPuntTimeout != 0) {
2898 actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2900 instructions.add(new InstructionApplyActions(actionsInfos));
2902 String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2903 FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2904 flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2905 LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2909 public void installNaptPfibEntryWithBgpVpn(Uint64 dpnId, Uint32 segmentId, Uint32 changedVpnId,
2910 TypedWriteTransaction<Configuration> writeFlowInvTx) {
2911 LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2912 dpnId, segmentId, changedVpnId);
2913 FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2914 mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
2917 public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(Uint64 dpId, Uint32 segmentId, Uint32 changedVpnId) {
2919 LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2920 + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2921 List<MatchInfo> matches = new ArrayList<>();
2922 matches.add(MatchEthernetType.IPV4);
2923 matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2924 MetaDataUtil.METADATA_MASK_VRFID));
2926 ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2927 ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2928 listActionInfo.add(new ActionNxLoadInPort(Uint64.valueOf(BigInteger.ZERO)));
2929 listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2930 instructionInfo.add(new InstructionApplyActions(listActionInfo));
2932 String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2933 FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2934 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2935 NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2936 LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2941 protected ExternalRoutersListener getDataTreeChangeListener() {
2942 return ExternalRoutersListener.this;
2945 protected void installNaptPfibEntriesForExternalSubnets(String routerName, Uint64 dpnId,
2946 @Nullable TypedWriteTransaction<Configuration> writeFlowInvTx) {
2947 Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2949 for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2950 Uint32 subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2951 if (subnetVpnId != NatConstants.INVALID_ID) {
2952 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2953 + "and vpnId {}", dpnId, subnetVpnId);
2954 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);