2 * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow;
11 import static com.google.common.base.Preconditions.checkNotNull;
12 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP;
13 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4;
14 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6;
15 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch;
16 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns;
17 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction;
18 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch;
19 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum;
20 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.gotoTableIns;
21 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.groupAction;
22 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.instructions;
23 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpOpAction;
24 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpShaAction;
25 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction;
26 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction;
27 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action;
28 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction;
29 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction;
30 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction;
31 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction;
32 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction;
33 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction;
34 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction;
35 import static org.opendaylight.groupbasedpolicy.util.DataStoreHelper.readFromDs;
37 import java.math.BigInteger;
38 import java.util.ArrayList;
39 import java.util.Collection;
40 import java.util.Collections;
41 import java.util.HashMap;
42 import java.util.HashSet;
43 import java.util.List;
45 import java.util.Map.Entry;
46 import java.util.Objects;
49 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
50 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
51 import org.opendaylight.groupbasedpolicy.dto.EgKey;
52 import org.opendaylight.groupbasedpolicy.dto.EpKey;
53 import org.opendaylight.groupbasedpolicy.dto.IndexedTenant;
54 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext;
55 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfWriter;
56 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.endpoint.EndpointManager;
57 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch;
58 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals;
59 import org.opendaylight.groupbasedpolicy.util.IidFactory;
60 import org.opendaylight.groupbasedpolicy.util.TenantUtils;
61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix;
64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
65 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action;
66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
67 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
68 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
70 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match;
71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
74 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId;
75 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
76 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.SubnetId;
77 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
78 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints;
79 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address;
80 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.l3.prefix.fields.EndpointL3Gateways;
81 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
82 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
83 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
84 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Prefix;
85 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
86 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant;
87 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.ForwardingContext;
88 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L2FloodDomain;
89 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L3Context;
90 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.Subnet;
91 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
92 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
93 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.ethernet.match.fields.EthernetDestinationBuilder;
94 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatchBuilder;
95 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.Layer3Match;
96 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder;
97 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder;
98 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder;
99 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2;
100 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3;
101 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4;
102 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5;
103 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
104 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7;
105 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
106 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
107 import org.slf4j.Logger;
108 import org.slf4j.LoggerFactory;
110 import com.google.common.base.Optional;
111 import com.google.common.base.Strings;
112 import com.google.common.collect.HashMultimap;
113 import com.google.common.collect.SetMultimap;
114 import com.google.common.collect.Sets;
117 * Manage the table that maps the destination address to the next hop for the
118 * path as well as applies any relevant routing transformations.
120 public class DestinationMapper extends FlowTable {
122 protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class);
124 // TODO Li alagalah: Improve UT coverage for this class.
126 // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows
127 // This ensures we have the appropriate network-containment'
129 public static short TABLE_ID;
131 * This is the MAC address of the magical router in the sky
133 public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5");
134 public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00");
135 public static final Integer BASE_L3_PRIORITY = 100;
137 public DestinationMapper(OfContext ctx, short tableId) {
139 this.TABLE_ID = tableId;
142 Map<TenantId, HashSet<Subnet>> subnetsByTenant = new HashMap<TenantId, HashSet<Subnet>>();
145 public short getTableId() {
150 public void sync(NodeId nodeId, OfWriter ofWriter) throws Exception {
152 TenantId currentTenant = null;
154 ofWriter.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null, TABLE_ID));
156 SetMultimap<EpKey, EpKey> visitedEps = HashMultimap.create();
157 Set<EndpointFwdCtxOrdinals> epOrdSet = new HashSet<>();
159 for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) {
160 Set<EndpointGroupId> srcEpgIds = new HashSet<>();
161 if (srcEp.getEndpointGroup() != null)
162 srcEpgIds.add(srcEp.getEndpointGroup());
163 if (srcEp.getEndpointGroups() != null)
164 srcEpgIds.addAll(srcEp.getEndpointGroups());
166 for (EndpointGroupId epgId : srcEpgIds) {
167 EgKey epg = new EgKey(srcEp.getTenant(), epgId);
168 Set<EgKey> peers = Sets.union(Collections.singleton(epg), ctx.getCurrentPolicy().getPeers(epg));
169 for (EgKey peer : peers) {
170 Collection<Endpoint> endpointsForGroup = new HashSet<>();
171 endpointsForGroup.addAll(ctx.getEndpointManager().getEndpointsForGroup(peer));
172 endpointsForGroup.addAll(ctx.getEndpointManager().getExtEpsNoLocForGroup(peer));
173 for (Endpoint peerEp : endpointsForGroup) {
174 currentTenant = peerEp.getTenant();
175 subnetsByTenant.put(currentTenant, getSubnets(currentTenant));
176 EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress());
177 EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress());
179 if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) {
182 syncEP(ofWriter, nodeId, srcEp, peerEp);
183 visitedEps.put(srcEpKey, peerEpKey);
185 // Process subnets and flood-domains for epPeer
186 EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx,
188 if (epOrds == null) {
189 LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", peerEp);
193 epOrdSet.add(epOrds);
199 for (Entry<TenantId, HashSet<Subnet>> subnetEntry : subnetsByTenant.entrySet()) {
200 if (subnetEntry.getValue() == null) {
201 LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey());
204 currentTenant = subnetEntry.getKey();
205 for (Subnet sn : subnetEntry.getValue()) {
206 L3Context l3c = getL3ContextForSubnet(currentTenant, sn);
207 Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn,
208 OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId()));
209 if (arpFlow != null) {
210 ofWriter.writeFlow(nodeId, TABLE_ID, arpFlow);
213 "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .",
214 sn.getIpPrefix().getValue());
219 // Write broadcast flows per flood domain.
220 for (EndpointFwdCtxOrdinals epOrd : epOrdSet) {
221 if (ofWriter.groupExists(nodeId, Integer.valueOf(epOrd.getFdId()).longValue())) {
222 ofWriter.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd));
226 // L3 Prefix Endpoint handling
227 Collection<EndpointL3Prefix> prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(currentTenant);
228 if (prefixEps != null) {
229 LOG.trace("DestinationMapper - Processing L3PrefixEndpoints");
230 for (EndpointL3Prefix prefixEp : prefixEps) {
231 List<Subnet> localSubnets = getLocalSubnets(nodeId);
232 if (localSubnets == null) {
235 for (Subnet localSubnet: localSubnets) {
236 Flow prefixFlow = createL3PrefixFlow(prefixEp, nodeId, localSubnet);
237 if (prefixFlow != null) {
238 ofWriter.writeFlow(nodeId, TABLE_ID, prefixFlow);
239 LOG.trace("Wrote L3Prefix flow");
248 // set up next-hop destinations for all the endpoints in the endpoint
251 private Flow createL3PrefixFlow(EndpointL3Prefix prefixEp, NodeId nodeId, Subnet subnet) throws Exception {
253 * Priority: 100+lengthprefix
254 * Match: prefix, l3c, "mac address of router" ?
256 * - set Reg2, Reg3 for L3Ep by L2Ep ?
258 * - Reg7: use switch location external port else punt for now
260 * - Reg7: grab L2Ep from L3Ep and use its location info
261 * - goto_table: POLENF (will check there for external on EP)
264 ReadOnlyTransaction rTx = ctx.getDataBroker().newReadOnlyTransaction();
265 // TODO Bug #3440 Target: Be - should support for more than first gateway.
266 EndpointL3Gateways l3Gateway = prefixEp.getEndpointL3Gateways().get(0);
267 Optional<EndpointL3> optL3Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
268 IidFactory.l3EndpointIid(l3Gateway.getL3Context(), l3Gateway.getIpAddress()), rTx);
269 if (!optL3Ep.isPresent()) {
270 LOG.error("createL3PrefixFlow - L3Endpoint gateway {} for L3Prefix {} not found.", l3Gateway, prefixEp);
273 EndpointL3 l3Ep = optL3Ep.get();
274 if (l3Ep.getL2Context() == null || l3Ep.getMacAddress() == null) {
275 LOG.debug("L3 endpoint representing L3 gateway does not contain L2-context or MAC address. {}", l3Ep);
278 Optional<Endpoint> optL2Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
279 IidFactory.endpointIid(l3Ep.getL2Context(), l3Ep.getMacAddress()), rTx);
280 if (!optL2Ep.isPresent()) {
281 LOG.error("createL3PrefixFlow - L2Endpoint for L3Gateway {} not found.", l3Ep);
284 Endpoint l2Ep = optL2Ep.get();
285 EndpointFwdCtxOrdinals epFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, l2Ep);
286 if (epFwdCtxOrds == null) {
287 LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", l2Ep);
291 NetworkDomainId epNetworkContainment = getEPNetworkContainment(l2Ep);
293 MacAddress epDestMac = l2Ep.getMacAddress();
294 MacAddress destSubnetGatewayMac = l2Ep.getMacAddress();
295 L3Context destL3c = getL3ContextForSubnet(prefixEp.getTenant(), subnet);
296 if (destL3c == null || destL3c.getId() == null) {
297 LOG.error("No L3 Context found associated with subnet {}", subnet.getId());
301 MacAddress matcherMac = routerPortMac(destL3c, subnet.getVirtualRouterIp());
303 ArrayList<Instruction> l3instructions = new ArrayList<>();
304 List<Action> applyActions = new ArrayList<>();
305 List<Action> l3ApplyActions = new ArrayList<>();
309 Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
310 Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
314 OfOverlayContext ofc = l2Ep.getAugmentation(OfOverlayContext.class);
317 if (EndpointManager.isInternal(l2Ep, ctx.getTenant(l2Ep.getTenant()).getExternalImplicitGroups())) {
318 checkNotNull(ofc.getNodeConnectorId());
319 nextHop = ofc.getNodeConnectorId().getValue();
321 portNum = getOfPortNum(ofc.getNodeConnectorId());
322 } catch (NumberFormatException ex) {
323 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
329 Set<NodeConnectorId> externalPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
330 checkNotNull(externalPorts);
331 for (NodeConnectorId externalPort : externalPorts) {
332 // TODO Bug #3440 Target: Be - should support for more than first external port.
333 //TODO Bug 3546 - Difficult: External port is unrelated to Tenant, L3C, L2BD..
334 nextHop = externalPort.getValue();
336 portNum = getOfPortNum(externalPort);
337 } catch (NumberFormatException ex) {
338 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
345 if (Strings.isNullOrEmpty(nextHop)
347 LOG.error("createL3Prefix - Cannot find nodeConnectorId for {} for Prefix: ", l2Ep, prefixEp);
350 setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
352 Action setDlDst = setDlDstAction(epDestMac);
353 l3ApplyActions.add(setDlDst);
355 Action decTtl = decNwTtlAction();
356 l3ApplyActions.add(decTtl);
359 applyActions.add(setdEPG);
360 applyActions.add(setdCG);
361 applyActions.add(setNextHop);
363 applyActions.addAll(l3ApplyActions);
364 Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
365 .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
368 l3instructions.add(applyActionsIns);
369 Instruction gotoTable = new InstructionBuilder().setOrder(order++)
370 .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
372 l3instructions.add(gotoTable);
374 Layer3Match m = null;
375 Long etherType = null;
377 Integer prefixLength=0;
378 if (prefixEp.getIpPrefix().getIpv4Prefix() != null) {
379 ikey = prefixEp.getIpPrefix().getIpv4Prefix().getValue();
381 prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv4Prefix().getValue().split("/")[1]);
382 m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
383 } else if (prefixEp.getIpPrefix().getIpv6Prefix() != null) {
384 ikey = prefixEp.getIpPrefix().getIpv6Prefix().getValue();
387 * This will result in flows with priority between 100-228, but since its matching on IPv6 prefix as well
388 * this shouldn't pose and issue, as the priority is more important within the address space of the matcher,
389 * even though technically flows are processed in priority order.
392 prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv6Prefix().getValue().split("/")[1]);
393 m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
395 LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", prefixEp);
399 MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType));
400 addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
401 Match match = mb.build();
402 FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "L3prefix", match);
403 FlowBuilder flowb = base().setId(flowid)
404 .setPriority(Integer.valueOf(BASE_L3_PRIORITY+prefixLength))
406 .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
407 return flowb.build();
410 private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) {
411 MatchBuilder mb = new MatchBuilder()
412 .setEthernetMatch(new EthernetMatchBuilder().setEthernetDestination(
413 new EthernetDestinationBuilder().setAddress(MULTICAST_MAC)
414 .setMask(MULTICAST_MAC)
416 addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId())));
418 Match match = mb.build();
419 FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "broadcast", match);
420 FlowBuilder flowb = base().setPriority(Integer.valueOf(140))
424 instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false),
425 groupAction(Long.valueOf(epOrd.getFdId())))));
427 return flowb.build();
430 private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) {
432 if (ctx.getDataBroker() == null) {
436 MacAddress defaultMacAddress = ROUTER_MAC;
438 EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId());
439 InstanceIdentifier<EndpointL3> endpointsIid = InstanceIdentifier.builder(Endpoints.class)
440 .child(EndpointL3.class, l3Key)
442 ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
444 Optional<EndpointL3> r;
446 r = t.read(LogicalDatastoreType.OPERATIONAL, endpointsIid).get();
448 return defaultMacAddress;
449 EndpointL3 epL3 = r.get();
450 if (epL3.getMacAddress() == null) {
451 return defaultMacAddress;
453 return epL3.getMacAddress();
455 } catch (Exception e) {
456 LOG.error("Error reading EndpointL3 {}.{}", l3c, ipAddress, e);
461 private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) {
462 IndexedTenant indexedTenant = ctx.getTenant(tenantId);
463 if (indexedTenant == null) {
464 LOG.debug("Tenant {} is null, cannot get L3 context", tenantId);
467 L3Context l3c = indexedTenant.resolveL3Context(sn.getId());
471 private Flow createRouterArpFlow(TenantId tenantId, NodeId nodeId, Subnet sn, int l3Id) {
472 if (sn == null || sn.getVirtualRouterIp() == null) {
473 LOG.trace("Didn't create routerArpFlow since either subnet or subnet virtual router was null");
477 * TODO: Li alagalah: This should be new Yang "gateways" list as well,
478 * that expresses the gateway and prefixes it is interface for. Should
479 * also check for external.
481 if (sn.getVirtualRouterIp().getIpv4Address() != null) {
482 String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue();
484 L3Context l3c = getL3ContextForSubnet(tenantId, sn);
486 LOG.error("No L3 Context found associated with subnet {}", sn.getId());
489 MacAddress routerMac = routerPortMac(l3c, sn.getVirtualRouterIp());
490 if (routerMac == null) {
494 BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue()));
496 MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match(
497 new ArpMatchBuilder().setArpOp(Integer.valueOf(1))
498 .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32"))
500 addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id)));
502 Match match = mb.build();
503 FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "routerarp", match);
504 FlowBuilder flowb = base().setPriority(150)
508 instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac),
509 nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(),
510 nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(),
511 nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue()
513 return flowb.build();
515 LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId()
522 private Flow createLocalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) {
524 // TODO Li alagalah - refactor common code but keep simple method
525 ArrayList<Instruction> instructions = new ArrayList<>();
526 List<Action> applyActions = new ArrayList<>();
530 Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
531 Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
536 nextHop = ofc.getNodeConnectorId().getValue();
540 portNum = getOfPortNum(ofc.getNodeConnectorId());
541 } catch (NumberFormatException ex) {
542 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
546 setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
551 applyActions.add(setdEPG);
552 applyActions.add(setdCG);
553 applyActions.add(setNextHop);
554 Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
555 .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
557 instructions.add(applyActionsIns);
559 Instruction gotoTable = new InstructionBuilder().setOrder(order++)
560 .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
562 instructions.add(gotoTable);
564 MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
565 addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
566 Match match = mb.build();
567 FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL2", match);
568 FlowBuilder flowb = base().setId(flowid)
569 .setPriority(Integer.valueOf(50))
571 .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
572 return flowb.build();
575 private void syncEP(OfWriter ofWriter, NodeId nodeId, Endpoint srcEp, Endpoint destEp)
578 if (ctx.getTenant(srcEp.getTenant()) == null
579 || ctx.getTenant(destEp.getTenant()) == null) {
580 LOG.debug("Source or destination EP references empty tenant srcEp:{} destEp:{}", srcEp, destEp);
584 // TODO: Conditions messed up, but for now, send policyInfo until this
586 EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, destEp);
587 if (destEpFwdCtxOrds == null) {
588 LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", destEp);
591 EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, srcEp);
592 if (srcEpFwdCtxOrds == null) {
593 LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", srcEp);
598 if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) {
599 if (destEp.getTenant() == null) {
600 LOG.debug("Didn't process endpoint {} due to tenant being null", destEp.getKey());
602 LOG.debug("Didn't process endpoint {} due to EPG(s) being null", destEp.getKey());
608 * Only care about subnets for L3, but fetch them before loop. We need
609 * the local subnets for setting SRC MAC for routing. All Routing is now
610 * done locally! YAY! Instead of being shovelled L2 style across network
613 List<Subnet> localSubnets = getLocalSubnets(nodeId);
614 if (localSubnets == null) {
615 LOG.error("No subnets could be found locally for node: {}", nodeId);
619 OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class);
621 // forwarding outside of internal domain should be done when dest EP or GW is external.
622 Subnet srcSubnet = ctx.getTenant(srcEp.getTenant()).resolveSubnet(new SubnetId(srcEp.getNetworkContainment()));
623 Endpoint l2Gw = this.getL2EndpointOfSubnetGateway(srcEp.getTenant(), srcSubnet);
624 boolean destEpIsExternal = destEp.getNetworkContainment() != null
625 && EndpointManager.isExternal(destEp, ctx.getTenant(destEp.getTenant()).getExternalImplicitGroups());
626 boolean subnetGwIsExternal = l2Gw != null
627 && EndpointManager.isExternal(l2Gw, ctx.getTenant(l2Gw.getTenant()).getExternalImplicitGroups());
628 if (destEpIsExternal || subnetGwIsExternal) {
629 if (ofc == null && destEp.getNetworkContainment().equals(srcEp.getNetworkContainment())) {
630 Flow flow = createExternalL2Flow(destEp, destEpFwdCtxOrds, nodeId);
632 ofWriter.writeFlow(nodeId, TABLE_ID, flow);
634 } else if (l2Gw != null && EndpointManager.isExternal(l2Gw, ctx.getTenant(l2Gw.getTenant()).getExternalImplicitGroups())
635 && !destEp.getNetworkContainment().equals(srcEp.getNetworkContainment())) {
636 for (L3Address l3a : destEp.getL3Address()) {
637 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
638 LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
639 destEp.getL3Address());
642 for (Subnet localSubnet : localSubnets) {
643 Flow extL3Flow = createExternalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, localSubnet, nodeId);
644 if (extL3Flow != null) {
645 ofWriter.writeFlow(nodeId, TABLE_ID, extL3Flow);
647 LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
648 localSubnet.getIpPrefix().getValue());
654 else if (ofc != null && Objects.equals(ofc.getNodeId(), nodeId)) {
655 // this is a local endpoint; send to the approppriate local
658 if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
659 ofWriter.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc));
661 // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
662 // The Endpoint conflation must end!
663 if (destEp.getL3Address() == null) {
664 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
668 for (L3Address l3a : destEp.getL3Address()) {
669 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
670 LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
671 destEp.getL3Address());
674 for (Subnet localSubnet : localSubnets) {
675 Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet);
677 ofWriter.writeFlow(nodeId, TABLE_ID, flow);
679 LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
680 localSubnet.getIpPrefix().getValue());
685 } else if(ofc!= null) {
686 // this endpoint is on a different switch; send to the
687 // appropriate tunnel
688 if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
689 Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc);
690 if (remoteL2Flow != null) {
691 ofWriter.writeFlow(nodeId, TABLE_ID, remoteL2Flow);
694 LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}",
695 srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId());
698 // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
699 // The Endpoint conflation must end!
700 if (destEp.getL3Address() == null) {
701 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
704 for (L3Address l3a : destEp.getL3Address()) {
705 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
706 LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
707 destEp.getL3Address());
710 for (Subnet localSubnet : localSubnets) {
711 Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds,
712 destEpFwdCtxOrds, ofc, localSubnet);
713 if (remoteL3Flow != null) {
714 ofWriter.writeFlow(nodeId, TABLE_ID, remoteL3Flow);
716 LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
717 localSubnet.getIpPrefix().getValue());
729 * ################################## DestMapper Flow methods
730 * ##################################
732 private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
733 OfOverlayContext ofc, Subnet srcSubnet) {
735 // TODO Li alagalah - refactor common code but keep simple method
737 Subnet destSubnet = null;
738 HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
739 if (subnets == null) {
740 LOG.trace("No subnets in tenant {}", destEp.getTenant());
743 NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
744 for (Subnet subnet : subnets) {
745 // TODO Li alagalah add IPv6 support
746 if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
751 if (destSubnet == null) {
752 LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
756 if (destSubnet.getVirtualRouterIp() == null) {
757 LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
758 destL3Address.getKey());
762 if (srcSubnet.getVirtualRouterIp() == null) {
763 LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
766 L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
767 if (destL3c == null || destL3c.getId() == null) {
768 LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
771 L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
772 if (srcL3c == null || srcL3c.getId() == null) {
773 LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
777 if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
778 LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
779 .getValue(), destL3c.getId().getValue());
783 MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
784 MacAddress epDestMac = destEp.getMacAddress();
785 MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
787 if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
788 // This is our final destination, so match on actual EP mac.
789 matcherMac = epDestMac;
792 ArrayList<Instruction> l3instructions = new ArrayList<>();
793 List<Action> applyActions = new ArrayList<>();
794 List<Action> l3ApplyActions = new ArrayList<>();
798 Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
799 Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
804 nextHop = ofc.getNodeConnectorId().getValue();
808 portNum = getOfPortNum(ofc.getNodeConnectorId());
809 } catch (NumberFormatException ex) {
810 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
814 setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
817 // Lets not re-write the srcMac if its local.
818 if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
819 Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
820 l3ApplyActions.add(setDlSrc);
823 Action setDlDst = setDlDstAction(epDestMac);
824 l3ApplyActions.add(setDlDst);
826 Action decTtl = decNwTtlAction();
827 l3ApplyActions.add(decTtl);
830 applyActions.add(setdEPG);
831 applyActions.add(setdCG);
832 applyActions.add(setNextHop);
834 applyActions.addAll(l3ApplyActions);
835 Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
836 .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
839 l3instructions.add(applyActionsIns);
840 Instruction gotoTable = new InstructionBuilder().setOrder(order++)
841 .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
843 l3instructions.add(gotoTable);
844 Layer3Match m = null;
845 Long etherType = null;
847 if (destL3Address.getIpAddress().getIpv4Address() != null) {
848 ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
850 m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
851 } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
852 ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
854 m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
856 LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
860 MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
862 addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
863 Match match = mb.build();
864 FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL3", match);
865 FlowBuilder flowb = base().setId(flowid)
866 .setPriority(Integer.valueOf(132))
868 .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
869 return flowb.build();
872 private Flow createExternalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
873 Subnet srcSubnet, NodeId nodeId) {
875 Subnet destSubnet = null;
876 HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
877 if (subnets == null) {
878 LOG.trace("No subnets in tenant {}", destEp.getTenant());
881 NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
882 for (Subnet subnet : subnets) {
883 // TODO Li alagalah add IPv6 support
884 if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
889 if (destSubnet == null) {
890 LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
894 if (destSubnet.getVirtualRouterIp() == null) {
895 LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
896 destL3Address.getKey());
900 if (srcSubnet.getVirtualRouterIp() == null) {
901 LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
904 L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
905 if (destL3c == null || destL3c.getId() == null) {
906 LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
909 L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
910 if (srcL3c == null || srcL3c.getId() == null) {
911 LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
915 if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
916 LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
917 .getValue(), destL3c.getId().getValue());
921 Endpoint l2Gw = getL2EndpointOfSubnetGateway(destEp.getTenant(), srcSubnet);
923 LOG.warn("The endpoint representing external gateway of subnet {} not found", srcSubnet);
926 MacAddress matcherMac = destEp.getMacAddress();
927 MacAddress destSubnetGatewayMac = l2Gw.getMacAddress();
929 ArrayList<Instruction> l3instructions = new ArrayList<>();
930 List<Action> applyActions = new ArrayList<>();
931 List<Action> l3ApplyActions = new ArrayList<>();
935 Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
936 Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
939 Set<NodeConnectorId> extPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
940 if (extPorts == null || !extPorts.iterator().hasNext()) {
941 LOG.warn("No external interface on node: {}. External Gateway {} is not reachable!", nodeId, l2Gw.getKey());
944 // only one external port is supported for now
945 NodeConnectorId extPort = extPorts.iterator().next();
949 portNum = getOfPortNum(extPort);
950 } catch (NumberFormatException ex) {
951 LOG.warn("Could not parse port number {}", extPort, ex);
955 setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
959 Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
960 l3ApplyActions.add(setDlSrc);
962 Action setDlDst = setDlDstAction(l2Gw.getMacAddress());
963 l3ApplyActions.add(setDlDst);
966 applyActions.add(setdEPG);
967 applyActions.add(setdCG);
968 applyActions.add(setNextHop);
970 applyActions.addAll(l3ApplyActions);
971 Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
972 .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
975 l3instructions.add(applyActionsIns);
976 Instruction gotoTable = new InstructionBuilder().setOrder(order++)
977 .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
979 l3instructions.add(gotoTable);
980 Layer3Match m = null;
981 Long etherType = null;
983 if (destL3Address.getIpAddress().getIpv4Address() != null) {
984 ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
986 m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
987 } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
988 ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
990 m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
992 LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
996 MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
998 addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
999 Match match = mb.build();
1000 FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "externalL3", match);
1001 FlowBuilder flowb = base().setId(flowid)
1002 .setPriority(Integer.valueOf(132))
1004 .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
1005 return flowb.build();
1008 private Endpoint getL2EndpointOfSubnetGateway(TenantId tenantId, Subnet subnet) {
1009 if (subnet != null && subnet.getVirtualRouterIp() != null) {
1010 IpAddress gwIpAddress = subnet.getVirtualRouterIp();
1011 Collection<EndpointL3Prefix> prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(tenantId);
1012 if (prefixEps != null) {
1013 for (EndpointL3Prefix prefixEp : prefixEps) {
1014 for (EndpointL3Gateways gw : prefixEp.getEndpointL3Gateways()) {
1015 EndpointL3 l3Ep = ctx.getEndpointManager().getL3Endpoint(gw.getL3Context(), gwIpAddress,
1016 prefixEp.getTenant());
1017 if (l3Ep != null && l3Ep.getL2Context() != null && l3Ep.getMacAddress() != null) {
1018 return ctx.getEndpointManager().getEndpoint(
1019 new EpKey(l3Ep.getL2Context(), l3Ep.getMacAddress()));
1028 private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds,
1029 EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) {
1031 // TODO Li alagalah - refactor common code but keep simple method
1033 // this endpoint is on a different switch; send to the
1034 // appropriate tunnel
1036 ArrayList<Instruction> instructions = new ArrayList<>();
1037 List<Action> applyActions = new ArrayList<>();
1041 Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
1042 Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
1046 // BEGIN TUNNEL HANDLING
1047 IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
1048 NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
1049 if (tunDst == null) {
1050 LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep);
1053 if (tunPort == null) {
1054 LOG.warn("Failed to get Tunnel Port for NodeId {} with EP {}", nodeId, ep);
1058 Action tundstAction;
1060 if (tunDst.getIpv4Address() != null) {
1061 nextHop = tunDst.getIpv4Address().getValue();
1062 tundstAction = nxLoadTunIPv4Action(nextHop, false);
1063 } else if (tunDst.getIpv6Address() != null) {
1064 // nextHop = tunDst.getIpv6Address().getValue();
1065 LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
1069 // this shouldn't happen
1070 LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
1076 portNum = getOfPortNum(tunPort);
1077 } catch (NumberFormatException ex) {
1078 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
1082 setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1083 applyActions.add(tundstAction);
1087 applyActions.add(setdEPG);
1088 applyActions.add(setdCG);
1089 applyActions.add(setNextHop);
1090 Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1091 .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1093 instructions.add(applyActionsIns);
1095 applyActionsIns = new InstructionBuilder().setOrder(order++)
1096 .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1099 Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1100 .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1102 instructions.add(gotoTable);
1104 MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
1105 addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId())));
1106 Match match = mb.build();
1107 FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL2", match);
1108 FlowBuilder flowb = base().setId(flowid)
1109 .setPriority(Integer.valueOf(50))
1111 .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
1113 return flowb.build();
1116 private Flow createExternalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds,NodeId nodeId) {
1118 ArrayList<Instruction> instructions = new ArrayList<>();
1119 List<Action> applyActions = new ArrayList<>();
1123 Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
1124 Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
1128 Set<NodeConnectorId> extPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
1129 if(extPorts == null || !extPorts.iterator().hasNext()) {
1132 // Only one external port is currently supported.
1133 NodeConnectorId extPort = extPorts.iterator().next();
1136 portNum = getOfPortNum(extPort);
1137 } catch (NumberFormatException ex) {
1138 LOG.warn("Could not parse port number {}", extPort, ex);
1141 setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1146 applyActions.add(setdEPG);
1147 applyActions.add(setdCG);
1148 applyActions.add(setNextHop);
1149 Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1150 .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1152 instructions.add(applyActionsIns);
1154 Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1155 .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1157 instructions.add(gotoTable);
1159 MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
1160 addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
1161 Match match = mb.build();
1162 FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "externalL2", match);
1163 FlowBuilder flowb = base().setId(flowid)
1164 .setPriority(Integer.valueOf(50))
1166 .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
1167 return flowb.build();
1170 private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId,
1171 EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc,
1174 // TODO Li alagalah - refactor common code but keep simple method
1176 // this endpoint is on a different switch; send to the
1177 // appropriate tunnel
1178 Subnet destSubnet = null;
1179 HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
1180 if (subnets == null) {
1181 LOG.trace("No subnets in tenant {}", destEp.getTenant());
1184 NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
1185 for (Subnet subnet : subnets) {
1186 // TODO Li alagalah add IPv6 support
1187 if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
1188 destSubnet = subnet;
1192 if (destSubnet == null) {
1193 LOG.info("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
1197 if (destSubnet.getVirtualRouterIp() == null) {
1198 LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
1199 destL3Address.getKey());
1203 if (srcSubnet.getVirtualRouterIp() == null) {
1204 LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
1207 L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
1208 if (destL3c == null || destL3c.getId() == null) {
1209 LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
1212 L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
1213 if (srcL3c == null || srcL3c.getId() == null) {
1214 LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
1218 if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
1219 LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
1220 .getValue(), destL3c.getId().getValue());
1224 MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
1225 MacAddress epDestMac = destEp.getMacAddress();
1226 MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
1227 if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
1228 // This is our final destination, so match on actual EP mac.
1229 matcherMac = epDestMac;
1231 ArrayList<Instruction> l3instructions = new ArrayList<>();
1232 List<Action> applyActions = new ArrayList<>();
1233 List<Action> l3ApplyActions = new ArrayList<>();
1237 Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
1238 Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
1242 // BEGIN TUNNEL HANDLING
1243 IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
1244 NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
1245 if (tunDst == null) {
1246 LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address);
1249 if (tunPort == null) {
1250 LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, destL3Address);
1254 Action tundstAction;
1256 if (tunDst.getIpv4Address() != null) {
1257 nextHop = tunDst.getIpv4Address().getValue();
1258 tundstAction = nxLoadTunIPv4Action(nextHop, false);
1259 } else if (tunDst.getIpv6Address() != null) {
1260 // nextHop = tunDst.getIpv6Address().getValue();
1261 LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
1265 // this shouldn't happen
1266 LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
1272 portNum = getOfPortNum(tunPort);
1273 } catch (NumberFormatException ex) {
1274 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
1278 setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1279 applyActions.add(tundstAction);
1283 applyActions.add(setdEPG);
1284 applyActions.add(setdCG);
1285 applyActions.add(setNextHop);
1287 // Lets not re-write the srcMac if its local.
1288 if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
1289 Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
1290 l3ApplyActions.add(setDlSrc);
1293 Action setDlDst = setDlDstAction(epDestMac);
1294 l3ApplyActions.add(setDlDst);
1296 Action decTtl = decNwTtlAction();
1297 l3ApplyActions.add(decTtl);
1299 applyActions.addAll(l3ApplyActions);
1300 Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1301 .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1304 l3instructions.add(applyActionsIns);
1305 Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1306 .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1308 l3instructions.add(gotoTable);
1309 Layer3Match m = null;
1310 Long etherType = null;
1312 if (destL3Address.getIpAddress().getIpv4Address() != null) {
1313 ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
1315 m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
1316 } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
1317 ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
1319 m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
1321 LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
1325 MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
1327 addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id())));
1328 Match match = mb.build();
1329 FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL3", match);
1330 FlowBuilder flowb = base().setId(flowid)
1331 .setPriority(Integer.valueOf(132))
1333 .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
1334 return flowb.build();
1337 private NetworkDomainId getEPNetworkContainment(Endpoint endpoint) {
1338 if (endpoint.getNetworkContainment() != null) {
1339 return endpoint.getNetworkContainment();
1342 * TODO: Be alagalah: Endpoint Refactor: This should be set on input
1343 * which we can't do because of the backwards way endpoints were
1346 return ctx.getTenant(endpoint.getTenant())
1347 .getEndpointGroup(endpoint.getEndpointGroup())
1348 .getNetworkDomain();
1352 private HashSet<Subnet> getSubnets(final TenantId tenantId) {
1354 if (ctx.getDataBroker() == null) {
1358 ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
1359 InstanceIdentifier<Tenant> tiid = TenantUtils.tenantIid(tenantId);
1360 Optional<Tenant> tenantInfo;
1362 tenantInfo = t.read(LogicalDatastoreType.CONFIGURATION, tiid).get();
1363 } catch (Exception e) {
1364 LOG.error("Could not read Tenant {}", tenantId, e);
1370 if (!tenantInfo.isPresent()) {
1371 LOG.warn("Tenant {} not found", tenantId);
1375 ForwardingContext fwCtx = tenantInfo.get().getForwardingContext();
1376 if (fwCtx == null || fwCtx.getSubnet() == null) {
1377 return new HashSet<>();
1379 return new HashSet<>(fwCtx.getSubnet());
1382 // Need a method to get subnets for EPs attached to the node locally
1383 // to set the source Mac address for the router interface.
1384 private List<Subnet> getLocalSubnets(NodeId nodeId) {
1385 Collection<Endpoint> endpointsForNode = ctx.getEndpointManager().getEndpointsForNode(nodeId);
1387 List<Subnet> localSubnets = new ArrayList<Subnet>();
1389 for (Endpoint endpoint : endpointsForNode) {
1390 HashSet<Subnet> subnets = getSubnets(endpoint.getTenant());
1391 if (subnets == null) {
1392 LOG.debug("No local subnets in tenant {} for EP {}.", endpoint.getTenant(), endpoint.getKey());
1395 NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint);
1396 for (Subnet subnet : subnets) {
1397 if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) {
1398 localSubnets.add(subnet);
1402 return localSubnets;
1405 static byte[] bytesFromHexString(String values) {
1407 if (values != null) {
1410 String[] octets = target.split(":");
1412 byte[] ret = new byte[octets.length];
1413 for (int i = 0; i < octets.length; i++) {
1414 ret[i] = Integer.valueOf(octets[i], 16).byteValue();