X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=renderers%2Fofoverlay%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fgroupbasedpolicy%2Frenderer%2Fofoverlay%2Fflow%2FDestinationMapper.java;h=1e8fa2a28728e8b6057bb4252fb13decb2ac2928;hb=7b948a309cbb014f952e5f14a251773a5d846c57;hp=9f8e5a328ba07d17ad262ef76cf88420e14ed8e8;hpb=5045a1327bba703fd30861ddd96f0926e0354544;p=groupbasedpolicy.git diff --git a/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java b/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java old mode 100644 new mode 100755 index 9f8e5a328..1e8fa2a28 --- a/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java +++ b/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java @@ -8,12 +8,12 @@ package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow; +import static com.google.common.base.Preconditions.checkNotNull; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns; -import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.createNodePath; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum; @@ -32,6 +32,7 @@ import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtil import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction; +import static org.opendaylight.groupbasedpolicy.util.DataStoreHelper.readFromDs; import java.math.BigInteger; import java.util.ArrayList; @@ -46,53 +47,56 @@ import java.util.Objects; import java.util.Set; import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction; -import org.opendaylight.controller.md.sal.binding.api.ReadTransaction; import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType; -import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; -import org.opendaylight.groupbasedpolicy.endpoint.EpKey; +import org.opendaylight.groupbasedpolicy.dto.EgKey; +import org.opendaylight.groupbasedpolicy.dto.EpKey; +import org.opendaylight.groupbasedpolicy.dto.IndexedTenant; import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext; -import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.PolicyManager.FlowMap; +import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfWriter; +import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.endpoint.EndpointManager; import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch; import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals; -import org.opendaylight.groupbasedpolicy.resolver.EgKey; -import org.opendaylight.groupbasedpolicy.resolver.PolicyInfo; -import org.opendaylight.groupbasedpolicy.resolver.TenantUtils; +import org.opendaylight.groupbasedpolicy.util.IidFactory; +import org.opendaylight.groupbasedpolicy.util.TenantUtils; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress; import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action; -import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.go.to.table._case.GoToTable; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder; -import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.SubnetId; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.l3.prefix.fields.EndpointL3Gateways; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key; -import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.EndpointLocation.LocationType; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Prefix; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant; -import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.L3Context; -import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.Subnet; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.ForwardingContext; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L3Context; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.Subnet; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId; -import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.ethernet.match.fields.EthernetDestinationBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatchBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.Layer3Match; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder; +import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.dec.nw.ttl._case.DecNwTtl; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4; @@ -100,20 +104,98 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev14 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan; -import org.opendaylight.yangtools.yang.binding.DataObject; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Optional; +import com.google.common.base.Strings; import com.google.common.collect.HashMultimap; import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; -import com.google.common.util.concurrent.CheckedFuture; /** - * Manage the table that maps the destination address to the next hop for the - * path as well as applies any relevant routing transformations. + *

Manage the table that maps the destination address to the next hop for the + * path as well as applies any relevant routing transformations (table=3)

+ * + * Sync Ep flows, every endpoint pair creates L2 and L3 flow
+ * + * Also applies to L3 + *

+ * L2 Flows: + *

+ * External, local and remote L2 flows
+ * Priority = 50
+ * Matches:
+ * - dl_dst mac address {@link MacAddress}
+ * - loadReg4 {@link NxmNxReg4}
+ * Actions:
+ * - load tunnel Ipv4 (local and remote only)
+ * - loadReg2 {@link NxmNxReg2}
+ * - loadReg3 {@link NxmNxReg3}
+ * - loadReg7 (next hop) {@link NxmNxReg7}
+ * - {@link GoToTable} POLICY ENFORCER table
+ *

+ * L3 flows: + *

+ * External, local and remote L3 routed flows:
+ * Priority = 50
+ * Matches:
+ * - ip (ethertype) + * - dl_dst mac address {@link MacAddress}
+ * - setReg6 {@link NxmNxReg6}
+ * Actions:
+ * - loadReg2 {@link NxmNxReg2}
+ * - loadReg3 {@link NxmNxReg3}
+ * - loadReg4 (tunnel destination) {@link NxmNxReg4} (remote only)
+ * - loadReg7 (next hop) {@link NxmNxReg7}
+ * - set dst mac to eth_dst {@link MacAddress}
+ * - dec_ttl {@link DecNwTtl} (local only)
+ * - {@link GoToTable} POLICY ENFORCER table + *

+ * If virtual router ip is present in subnet, and subnet contains L3 context, arp flow is created
+ *

+ * Router Arp flow
+ * Priority = 150
+ * Matches:
+ * - arp (ethertype)
+ * - arp target transport address
+ * - setReg6 {@link NxmNxReg6}
+ * Actions:
+ * - move eth_src = eth_dst
+ * - set dl_src {@link MacAddress}
+ * - load arp_op
+ * - move arp_sha = arp_tha
+ * - load arp_sha
+ * - move arp_spa = arp_tpa
+ * - load arp_spa
+ * - output:port {@link NodeConnectorId}
+ *

+ * Broadcast flow (per flood domain) + * Priority = 140
+ * Matches:
+ * - ethernet destination {@link MacAddress} + * - setReg5 {@link NxmNxReg5}
+ * Actions:
+ * - load tunnel ID
+ * - group action
+ *

+ * L3 Prefix flow
+ * Priority = 140
+ * Matches:
+ * - ethernet destination {@link MacAddress} + * - setReg5 {@link NxmNxReg5}
+ * Actions:
+ * - dl_dst {@link MacAddress}
+ * - dec_ttl
+ * - loadReg2 {@link NxmNxReg2}
+ * - loadReg3 {@link NxmNxReg3}
+ * - loadReg4 (next hop) {@link NxmNxReg4}
+ * - loadReg7 (if internal, port_num == {@link NodeConnectorId of L2 EP} ) {@link NxmNxReg7}
+ * - loadReg7 (if external, port_num = external port) {@link NxmNxReg7}
+ * - {@link GoToTable} POLICY ENFORCER table */ public class DestinationMapper extends FlowTable { @@ -124,15 +206,17 @@ public class DestinationMapper extends FlowTable { // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows // This ensures we have the appropriate network-containment' - public static final short TABLE_ID = 2; + public static short TABLE_ID; /** * This is the MAC address of the magical router in the sky */ public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5"); public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00"); + public static final Integer BASE_L3_PRIORITY = 100; - public DestinationMapper(OfContext ctx) { + public DestinationMapper(OfContext ctx, short tableId) { super(ctx); + this.TABLE_ID = tableId; } Map> subnetsByTenant = new HashMap>(); @@ -143,11 +227,11 @@ public class DestinationMapper extends FlowTable { } @Override - public void sync(NodeId nodeId, PolicyInfo policyInfo, FlowMap flowMap) throws Exception { + public void sync(NodeId nodeId, OfWriter ofWriter) throws Exception { - TenantId currentTenant; + TenantId currentTenant = null; - flowMap.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null)); + ofWriter.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null, TABLE_ID)); SetMultimap visitedEps = HashMultimap.create(); Set epOrdSet = new HashSet<>(); @@ -161,9 +245,12 @@ public class DestinationMapper extends FlowTable { for (EndpointGroupId epgId : srcEpgIds) { EgKey epg = new EgKey(srcEp.getTenant(), epgId); - Set peers = Sets.union(Collections.singleton(epg), policyInfo.getPeers(epg)); + Set peers = Sets.union(Collections.singleton(epg), ctx.getCurrentPolicy().getPeers(epg)); for (EgKey peer : peers) { - for (Endpoint peerEp : ctx.getEndpointManager().getEndpointsForGroup(peer)) { + Collection endpointsForGroup = new HashSet<>(); + endpointsForGroup.addAll(ctx.getEndpointManager().getEndpointsForGroup(peer)); + endpointsForGroup.addAll(ctx.getEndpointManager().getExtEpsNoLocForGroup(peer)); + for (Endpoint peerEp : endpointsForGroup) { currentTenant = peerEp.getTenant(); subnetsByTenant.put(currentTenant, getSubnets(currentTenant)); EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress()); @@ -172,12 +259,17 @@ public class DestinationMapper extends FlowTable { if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) { continue; } - syncEP(flowMap, nodeId, policyInfo, srcEp, peerEp); + syncEP(ofWriter, nodeId, srcEp, peerEp); visitedEps.put(srcEpKey, peerEpKey); // Process subnets and flood-domains for epPeer - EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, + EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, peerEp); + if (epOrds == null) { + LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", peerEp); + continue; + } + epOrdSet.add(epOrds); } } @@ -195,7 +287,7 @@ public class DestinationMapper extends FlowTable { Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn, OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId())); if (arpFlow != null) { - flowMap.writeFlow(nodeId, TABLE_ID, arpFlow); + ofWriter.writeFlow(nodeId, TABLE_ID, arpFlow); } else { LOG.debug( "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .", @@ -206,28 +298,208 @@ public class DestinationMapper extends FlowTable { // Write broadcast flows per flood domain. for (EndpointFwdCtxOrdinals epOrd : epOrdSet) { - if (groupExists(nodeId, epOrd.getFdId())) { - flowMap.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd)); + if (ofWriter.groupExists(nodeId, Integer.valueOf(epOrd.getFdId()).longValue())) { + ofWriter.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd)); + } + } + + // L3 Prefix Endpoint handling + Collection prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(currentTenant); + if (prefixEps != null) { + LOG.trace("DestinationMapper - Processing L3PrefixEndpoints"); + for (EndpointL3Prefix prefixEp : prefixEps) { + List localSubnets = getLocalSubnets(nodeId); + if (localSubnets == null) { + continue; + } + for (Subnet localSubnet: localSubnets) { + Flow prefixFlow = createL3PrefixFlow(prefixEp, nodeId, localSubnet); + if (prefixFlow != null) { + ofWriter.writeFlow(nodeId, TABLE_ID, prefixFlow); + LOG.trace("Wrote L3Prefix flow"); + } + } } } } + + // set up next-hop destinations for all the endpoints in the endpoint // group on the node + private Flow createL3PrefixFlow(EndpointL3Prefix prefixEp, NodeId nodeId, Subnet subnet) throws Exception { + /* + * Priority: 100+lengthprefix + * Match: prefix, l3c, "mac address of router" ? + * Action: + * - set Reg2, Reg3 for L3Ep by L2Ep ? + * - if external, + * - Reg7: use switch location external port else punt for now + * - if internal + * - Reg7: grab L2Ep from L3Ep and use its location info + * - goto_table: POLENF (will check there for external on EP) + */ + + ReadOnlyTransaction rTx = ctx.getDataBroker().newReadOnlyTransaction(); + // TODO Bug #3440 Target: Be - should support for more than first gateway. + EndpointL3Gateways l3Gateway = prefixEp.getEndpointL3Gateways().get(0); + Optional optL3Ep = readFromDs(LogicalDatastoreType.OPERATIONAL, + IidFactory.l3EndpointIid(l3Gateway.getL3Context(), l3Gateway.getIpAddress()), rTx); + if (!optL3Ep.isPresent()) { + LOG.error("createL3PrefixFlow - L3Endpoint gateway {} for L3Prefix {} not found.", l3Gateway, prefixEp); + return null; + } + EndpointL3 l3Ep = optL3Ep.get(); + if (l3Ep.getL2Context() == null || l3Ep.getMacAddress() == null) { + LOG.debug("L3 endpoint representing L3 gateway does not contain L2-context or MAC address. {}", l3Ep); + return null; + } + Optional optL2Ep = readFromDs(LogicalDatastoreType.OPERATIONAL, + IidFactory.endpointIid(l3Ep.getL2Context(), l3Ep.getMacAddress()), rTx); + if (!optL2Ep.isPresent()) { + LOG.error("createL3PrefixFlow - L2Endpoint for L3Gateway {} not found.", l3Ep); + return null; + } + Endpoint l2Ep = optL2Ep.get(); + EndpointFwdCtxOrdinals epFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, l2Ep); + if (epFwdCtxOrds == null) { + LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", l2Ep); + return null; + } + + NetworkDomainId epNetworkContainment = getEPNetworkContainment(l2Ep); + + MacAddress epDestMac = l2Ep.getMacAddress(); + MacAddress destSubnetGatewayMac = l2Ep.getMacAddress(); + L3Context destL3c = getL3ContextForSubnet(prefixEp.getTenant(), subnet); + if (destL3c == null || destL3c.getId() == null) { + LOG.error("No L3 Context found associated with subnet {}", subnet.getId()); + return null; + } + + MacAddress matcherMac = routerPortMac(destL3c, subnet.getVirtualRouterIp()); + + ArrayList l3instructions = new ArrayList<>(); + List applyActions = new ArrayList<>(); + List l3ApplyActions = new ArrayList<>(); + + int order = 0; + + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setNextHop; + String nextHop=null; + + OfOverlayContext ofc = l2Ep.getAugmentation(OfOverlayContext.class); + + long portNum = -1; + if (EndpointManager.isInternal(l2Ep, ctx.getTenant(l2Ep.getTenant()).getExternalImplicitGroups())) { + checkNotNull(ofc.getNodeConnectorId()); + nextHop = ofc.getNodeConnectorId().getValue(); + try { + portNum = getOfPortNum(ofc.getNodeConnectorId()); + } catch (NumberFormatException ex) { + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); + return null; + } + + } else { + // External + Set externalPorts = ctx.getSwitchManager().getExternalPorts(nodeId); + checkNotNull(externalPorts); + for (NodeConnectorId externalPort : externalPorts) { + // TODO Bug #3440 Target: Be - should support for more than first external port. + //TODO Bug 3546 - Difficult: External port is unrelated to Tenant, L3C, L2BD.. + nextHop = externalPort.getValue(); + try { + portNum = getOfPortNum(externalPort); + } catch (NumberFormatException ex) { + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); + return null; + } + continue; + } + } + + if (Strings.isNullOrEmpty(nextHop) + || portNum == -1) { + LOG.error("createL3Prefix - Cannot find nodeConnectorId for {} for Prefix: ", l2Ep, prefixEp); + return null; + } + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); + + Action setDlDst = setDlDstAction(epDestMac); + l3ApplyActions.add(setDlDst); + + Action decTtl = decNwTtlAction(); + l3ApplyActions.add(decTtl); + + order += 1; + applyActions.add(setdEPG); + applyActions.add(setdCG); + applyActions.add(setNextHop); + + applyActions.addAll(l3ApplyActions); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); + + l3instructions.add(applyActionsIns); + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER())) + .build(); + l3instructions.add(gotoTable); + + Layer3Match m = null; + Long etherType = null; + String ikey = null; + Integer prefixLength=0; + if (prefixEp.getIpPrefix().getIpv4Prefix() != null) { + ikey = prefixEp.getIpPrefix().getIpv4Prefix().getValue(); + etherType = IPv4; + prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv4Prefix().getValue().split("/")[1]); + m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build(); + } else if (prefixEp.getIpPrefix().getIpv6Prefix() != null) { + ikey = prefixEp.getIpPrefix().getIpv6Prefix().getValue(); + etherType = IPv6; + /* + * This will result in flows with priority between 100-228, but since its matching on IPv6 prefix as well + * this shouldn't pose and issue, as the priority is more important within the address space of the matcher, + * even though technically flows are processed in priority order. + */ + + prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv6Prefix().getValue().split("/")[1]); + m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build(); + } else { + LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", prefixEp); + return null; + } + + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType)); + addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id()))); + Match match = mb.build(); + FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "L3prefix", match); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(BASE_L3_PRIORITY+prefixLength)) + .setMatch(match) + .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build()); + return flowb.build(); + } + private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) { - FlowId flowId = new FlowId("broadcast|" + epOrd.getFdId()); MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(new EthernetMatchBuilder() - .setEthernetDestination(new EthernetDestinationBuilder(). - setAddress(MULTICAST_MAC) - .setMask(MULTICAST_MAC).build()) - .build()); + .setEthernetMatch(new EthernetMatchBuilder().setEthernetDestination( + new EthernetDestinationBuilder().setAddress(MULTICAST_MAC) + .setMask(MULTICAST_MAC) + .build()).build()); addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId()))); + Match match = mb.build(); + FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "broadcast", match); FlowBuilder flowb = base().setPriority(Integer.valueOf(140)) .setId(flowId) - .setMatch(mb.build()) + .setMatch(match) .setInstructions( instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false), groupAction(Long.valueOf(epOrd.getFdId()))))); @@ -235,32 +507,6 @@ public class DestinationMapper extends FlowTable { return flowb.build(); } - private boolean groupExists(NodeId nodeId, Integer fdId) throws Exception { - // Fetch existing GroupTables - if (ctx.getDataBroker() == null) { - return false; - } - - ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction(); - InstanceIdentifier niid = createNodePath(nodeId); - Optional r = t.read(LogicalDatastoreType.CONFIGURATION, niid).get(); - if (!r.isPresent()) - return false; - FlowCapableNode fcn = r.get().getAugmentation(FlowCapableNode.class); - if (fcn == null) - return false; - - if (fcn.getGroup() != null) { - for (Group g : fcn.getGroup()) { - if (g.getGroupId().getValue().equals(Long.valueOf(fdId))) { // Group - // Exists. - return true; - } - } - } - return false; - } - private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) { if (ctx.getDataBroker() == null) { @@ -293,7 +539,12 @@ public class DestinationMapper extends FlowTable { } private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) { - L3Context l3c = ctx.getPolicyResolver().getTenant(tenantId).resolveL3Context(sn.getId()); + IndexedTenant indexedTenant = ctx.getTenant(tenantId); + if (indexedTenant == null) { + LOG.debug("Tenant {} is null, cannot get L3 context", tenantId); + return null; + } + L3Context l3c = indexedTenant.resolveL3Context(sn.getId()); return l3c; } @@ -322,22 +573,17 @@ public class DestinationMapper extends FlowTable { BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue())); - FlowId flowId = new FlowId(new StringBuffer().append("routerarp|") - .append(sn.getId().getValue()) - .append("|") - .append(ikey) - .append("|") - .append(l3Id) - .toString()); MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match( new ArpMatchBuilder().setArpOp(Integer.valueOf(1)) .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32")) .build()); addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id))); + Match match = mb.build(); + FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "routerarp", match); FlowBuilder flowb = base().setPriority(150) .setId(flowId) - .setMatch(mb.build()) + .setMatch(match) .setInstructions( instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac), nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(), @@ -391,48 +637,50 @@ public class DestinationMapper extends FlowTable { instructions.add(applyActionsIns); Instruction gotoTable = new InstructionBuilder().setOrder(order++) - .setInstruction(gotoTableIns((short) (getTableId() + 1))) + .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER())) .build(); instructions.add(gotoTable); - FlowId flowid = new FlowId(new StringBuilder().append(epFwdCtxOrds.getBdId()) - .append("|l2|") - .append(ep.getMacAddress().getValue()) - .append("|") - .append(nextHop) - .toString()); MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null)); addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId()))); + Match match = mb.build(); + FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL2", match); FlowBuilder flowb = base().setId(flowid) .setPriority(Integer.valueOf(50)) - .setMatch(mb.build()) + .setMatch(match) .setInstructions(new InstructionsBuilder().setInstruction(instructions).build()); return flowb.build(); } - private void syncEP(FlowMap flowMap, NodeId nodeId, PolicyInfo policyInfo, Endpoint srcEp, Endpoint destEp) + private void syncEP(OfWriter ofWriter, NodeId nodeId, Endpoint srcEp, Endpoint destEp) throws Exception { + if (ctx.getTenant(srcEp.getTenant()) == null + || ctx.getTenant(destEp.getTenant()) == null) { + LOG.debug("Source or destination EP references empty tenant srcEp:{} destEp:{}", srcEp, destEp); + return; + } + // TODO: Conditions messed up, but for now, send policyInfo until this // is fixed. - EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, destEp); - EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, srcEp); - - if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) { - LOG.trace("Didn't process endpoint due to either tenant, or EPG(s) being null", destEp.getKey()); + EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, destEp); + if (destEpFwdCtxOrds == null) { + LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", destEp); + return; + } + EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, srcEp); + if (srcEpFwdCtxOrds == null) { + LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", srcEp); return; } - OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class); - // //////////////////////////////////////////////////////////////////////////////////////// - /* - * NOT HANDLING EXTERNALS TODO: alagalah Li: External Gateway - * functionality needed here. - */ - if (LocationType.External.equals(ofc.getLocationType())) { - // XXX - TODO - perform NAT and send to the external network - // TODO: Use case neutron gateway interface - LOG.warn("External endpoints not yet supported"); + + if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) { + if (destEp.getTenant() == null) { + LOG.debug("Didn't process endpoint {} due to tenant being null", destEp.getKey()); + } else { + LOG.debug("Didn't process endpoint {} due to EPG(s) being null", destEp.getKey()); + } return; } @@ -448,12 +696,47 @@ public class DestinationMapper extends FlowTable { return; } - if (Objects.equals(ofc.getNodeId(), nodeId)) { + OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class); + + // forwarding outside of internal domain should be done when dest EP or GW is external. + Subnet srcSubnet = ctx.getTenant(srcEp.getTenant()).resolveSubnet(new SubnetId(srcEp.getNetworkContainment())); + Endpoint l2Gw = this.getL2EndpointOfSubnetGateway(srcEp.getTenant(), srcSubnet); + boolean destEpIsExternal = destEp.getNetworkContainment() != null + && EndpointManager.isExternal(destEp, ctx.getTenant(destEp.getTenant()).getExternalImplicitGroups()); + boolean subnetGwIsExternal = l2Gw != null + && EndpointManager.isExternal(l2Gw, ctx.getTenant(l2Gw.getTenant()).getExternalImplicitGroups()); + if (destEpIsExternal || subnetGwIsExternal) { + if (ofc == null && destEp.getNetworkContainment().equals(srcEp.getNetworkContainment())) { + Flow flow = createExternalL2Flow(destEp, destEpFwdCtxOrds, nodeId); + if (flow != null) { + ofWriter.writeFlow(nodeId, TABLE_ID, flow); + } + } else if (l2Gw != null && EndpointManager.isExternal(l2Gw, ctx.getTenant(l2Gw.getTenant()).getExternalImplicitGroups()) + && !destEp.getNetworkContainment().equals(srcEp.getNetworkContainment())) { + for (L3Address l3a : destEp.getL3Address()) { + if (l3a.getIpAddress() == null || l3a.getL3Context() == null) { + LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}", + destEp.getL3Address()); + continue; + } + for (Subnet localSubnet : localSubnets) { + Flow extL3Flow = createExternalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, localSubnet, nodeId); + if (extL3Flow != null) { + ofWriter.writeFlow(nodeId, TABLE_ID, extL3Flow); + } else { + LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(), + localSubnet.getIpPrefix().getValue()); + } + } + } + } + } + else if (ofc != null && Objects.equals(ofc.getNodeId(), nodeId)) { // this is a local endpoint; send to the approppriate local // port if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) { - flowMap.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc)); + ofWriter.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc)); } // TODO Li alagalah: Need to move to EndpointL3 for L3 processing. // The Endpoint conflation must end! @@ -471,7 +754,7 @@ public class DestinationMapper extends FlowTable { for (Subnet localSubnet : localSubnets) { Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet); if (flow != null) { - flowMap.writeFlow(nodeId, TABLE_ID, flow); + ofWriter.writeFlow(nodeId, TABLE_ID, flow); } else { LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(), localSubnet.getIpPrefix().getValue()); @@ -479,13 +762,13 @@ public class DestinationMapper extends FlowTable { } } } - } else { + } else if(ofc!= null) { // this endpoint is on a different switch; send to the // appropriate tunnel if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) { Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc); if (remoteL2Flow != null) { - flowMap.writeFlow(nodeId, TABLE_ID, remoteL2Flow); + ofWriter.writeFlow(nodeId, TABLE_ID, remoteL2Flow); } } else { LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}", @@ -508,7 +791,7 @@ public class DestinationMapper extends FlowTable { Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc, localSubnet); if (remoteL3Flow != null) { - flowMap.writeFlow(nodeId, TABLE_ID, remoteL3Flow); + ofWriter.writeFlow(nodeId, TABLE_ID, remoteL3Flow); } else { LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(), localSubnet.getIpPrefix().getValue()); @@ -534,7 +817,7 @@ public class DestinationMapper extends FlowTable { Subnet destSubnet = null; HashSet subnets = getSubnets(destEp.getTenant()); if (subnets == null) { - LOG.trace("No subnets in tenant {}", destL3Address.getIpAddress()); + LOG.trace("No subnets in tenant {}", destEp.getTenant()); return null; } NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp); @@ -635,7 +918,143 @@ public class DestinationMapper extends FlowTable { l3instructions.add(applyActionsIns); Instruction gotoTable = new InstructionBuilder().setOrder(order++) - .setInstruction(gotoTableIns((short) (getTableId() + 1))) + .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER())) + .build(); + l3instructions.add(gotoTable); + Layer3Match m = null; + Long etherType = null; + String ikey = null; + if (destL3Address.getIpAddress().getIpv4Address() != null) { + ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32"; + etherType = IPv4; + m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build(); + } else if (destL3Address.getIpAddress().getIpv6Address() != null) { + ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128"; + etherType = IPv6; + m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build(); + } else { + LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString()); + return null; + } + + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType)) + .setLayer3Match(m); + addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id()))); + Match match = mb.build(); + FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL3", match); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(132)) + .setMatch(match) + .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build()); + return flowb.build(); + } + + private Flow createExternalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds, + Subnet srcSubnet, NodeId nodeId) { + + Subnet destSubnet = null; + HashSet subnets = getSubnets(destEp.getTenant()); + if (subnets == null) { + LOG.trace("No subnets in tenant {}", destEp.getTenant()); + return null; + } + NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp); + for (Subnet subnet : subnets) { + // TODO Li alagalah add IPv6 support + if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) { + destSubnet = subnet; + break; + } + } + if (destSubnet == null) { + LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress()); + return null; + } + + if (destSubnet.getVirtualRouterIp() == null) { + LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(), + destL3Address.getKey()); + return null; + } + + if (srcSubnet.getVirtualRouterIp() == null) { + LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix()); + return null; + } + L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet); + if (destL3c == null || destL3c.getId() == null) { + LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId()); + return null; + } + L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet); + if (srcL3c == null || srcL3c.getId() == null) { + LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId()); + return null; + } + + if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) { + LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId() + .getValue(), destL3c.getId().getValue()); + return null; + } + + Endpoint l2Gw = getL2EndpointOfSubnetGateway(destEp.getTenant(), srcSubnet); + if(l2Gw == null) { + LOG.warn("The endpoint representing external gateway of subnet {} not found", srcSubnet); + return null; + } + MacAddress matcherMac = destEp.getMacAddress(); + MacAddress destSubnetGatewayMac = l2Gw.getMacAddress(); + + ArrayList l3instructions = new ArrayList<>(); + List applyActions = new ArrayList<>(); + List l3ApplyActions = new ArrayList<>(); + + int order = 0; + + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setNextHop; + + Set extPorts = ctx.getSwitchManager().getExternalPorts(nodeId); + if (extPorts == null || !extPorts.iterator().hasNext()) { + LOG.warn("No external interface on node: {}. External Gateway {} is not reachable!", nodeId, l2Gw.getKey()); + return null; + } + // only one external port is supported for now + NodeConnectorId extPort = extPorts.iterator().next(); + + long portNum; + try { + portNum = getOfPortNum(extPort); + } catch (NumberFormatException ex) { + LOG.warn("Could not parse port number {}", extPort, ex); + return null; + } + + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); + // END L3 LOCAL + + + Action setDlSrc = setDlSrcAction(destSubnetGatewayMac); + l3ApplyActions.add(setDlSrc); + + Action setDlDst = setDlDstAction(l2Gw.getMacAddress()); + l3ApplyActions.add(setDlDst); + + order += 1; + applyActions.add(setdEPG); + applyActions.add(setdCG); + applyActions.add(setNextHop); + + applyActions.addAll(l3ApplyActions); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); + + l3instructions.add(applyActionsIns); + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER())) .build(); l3instructions.add(gotoTable); Layer3Match m = null; @@ -654,30 +1073,38 @@ public class DestinationMapper extends FlowTable { return null; } - FlowId flowid = new FlowId(new StringBuilder().append(Integer.toString(epFwdCtxOrds.getL3Id())) - .append("|l3|") - .append(ikey) - .append("|") - .append(Integer.toString(epFwdCtxOrds.getEpgId())) - .append("|") - .append(Integer.toString(epFwdCtxOrds.getCgId())) - .append("|") - .append(matcherMac) - .append("|") - .append(destSubnetGatewayMac) - .append("|") - .append(nextHop) - .toString()); MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType)) .setLayer3Match(m); addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id()))); + Match match = mb.build(); + FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "externalL3", match); FlowBuilder flowb = base().setId(flowid) .setPriority(Integer.valueOf(132)) - .setMatch(mb.build()) + .setMatch(match) .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build()); return flowb.build(); } + private Endpoint getL2EndpointOfSubnetGateway(TenantId tenantId, Subnet subnet) { + if (subnet != null && subnet.getVirtualRouterIp() != null) { + IpAddress gwIpAddress = subnet.getVirtualRouterIp(); + Collection prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(tenantId); + if (prefixEps != null) { + for (EndpointL3Prefix prefixEp : prefixEps) { + for (EndpointL3Gateways gw : prefixEp.getEndpointL3Gateways()) { + EndpointL3 l3Ep = ctx.getEndpointManager().getL3Endpoint(gw.getL3Context(), gwIpAddress, + prefixEp.getTenant()); + if (l3Ep != null && l3Ep.getL2Context() != null && l3Ep.getMacAddress() != null) { + return ctx.getEndpointManager().getEndpoint( + new EpKey(l3Ep.getL2Context(), l3Ep.getMacAddress())); + } + } + } + } + } + return null; + } + private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) { @@ -750,28 +1177,76 @@ public class DestinationMapper extends FlowTable { .build(); Instruction gotoTable = new InstructionBuilder().setOrder(order++) - .setInstruction(gotoTableIns((short) (getTableId() + 1))) + .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER())) .build(); instructions.add(gotoTable); - FlowId flowid = new FlowId(new StringBuilder().append(destEpFwdCtxOrds.getBdId()) - .append("|l2|") - .append(ep.getMacAddress().getValue()) - .append("|") - .append(srcEpFwdCtxOrds.getBdId()) - .append("|") - .append(nextHop) - .toString()); MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null)); addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId()))); + Match match = mb.build(); + FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL2", match); FlowBuilder flowb = base().setId(flowid) .setPriority(Integer.valueOf(50)) - .setMatch(mb.build()) + .setMatch(match) .setInstructions(new InstructionsBuilder().setInstruction(instructions).build()); return flowb.build(); } + private Flow createExternalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds,NodeId nodeId) { + + ArrayList instructions = new ArrayList<>(); + List applyActions = new ArrayList<>(); + + int order = 0; + + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setNextHop; + + // BEGIN L2 LOCAL + Set extPorts = ctx.getSwitchManager().getExternalPorts(nodeId); + if(extPorts == null || !extPorts.iterator().hasNext()) { + return null; + } + // Only one external port is currently supported. + NodeConnectorId extPort = extPorts.iterator().next(); + long portNum; + try { + portNum = getOfPortNum(extPort); + } catch (NumberFormatException ex) { + LOG.warn("Could not parse port number {}", extPort, ex); + return null; + } + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); + + // END L2 LOCAL + + order += 1; + applyActions.add(setdEPG); + applyActions.add(setdCG); + applyActions.add(setNextHop); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); + instructions.add(applyActionsIns); + + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER())) + .build(); + instructions.add(gotoTable); + + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null)); + addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId()))); + Match match = mb.build(); + FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "externalL2", match); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(50)) + .setMatch(match) + .setInstructions(new InstructionsBuilder().setInstruction(instructions).build()); + return flowb.build(); + } + private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc, Subnet srcSubnet) { @@ -783,7 +1258,7 @@ public class DestinationMapper extends FlowTable { Subnet destSubnet = null; HashSet subnets = getSubnets(destEp.getTenant()); if (subnets == null) { - LOG.trace("No subnets in tenant {}", destL3Address.getIpAddress()); + LOG.trace("No subnets in tenant {}", destEp.getTenant()); return null; } NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp); @@ -829,7 +1304,10 @@ public class DestinationMapper extends FlowTable { MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp()); MacAddress epDestMac = destEp.getMacAddress(); MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp()); - + if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) { + // This is our final destination, so match on actual EP mac. + matcherMac = epDestMac; + } ArrayList l3instructions = new ArrayList<>(); List applyActions = new ArrayList<>(); List l3ApplyActions = new ArrayList<>(); @@ -886,8 +1364,11 @@ public class DestinationMapper extends FlowTable { applyActions.add(setdCG); applyActions.add(setNextHop); - Action setDlSrc = setDlSrcAction(destSubnetGatewayMac); - l3ApplyActions.add(setDlSrc); + // Lets not re-write the srcMac if its local. + if (!(matcherMac.getValue().equals(epDestMac.getValue()))) { + Action setDlSrc = setDlSrcAction(destSubnetGatewayMac); + l3ApplyActions.add(setDlSrc); + } Action setDlDst = setDlDstAction(epDestMac); l3ApplyActions.add(setDlDst); @@ -902,7 +1383,7 @@ public class DestinationMapper extends FlowTable { l3instructions.add(applyActionsIns); Instruction gotoTable = new InstructionBuilder().setOrder(order++) - .setInstruction(gotoTableIns((short) (getTableId() + 1))) + .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER())) .build(); l3instructions.add(gotoTable); Layer3Match m = null; @@ -921,25 +1402,14 @@ public class DestinationMapper extends FlowTable { return null; } - FlowId flowid = new FlowId(new StringBuilder().append(Integer.toString(destEpFwdCtxOrds.getL3Id())) - .append("|l3|") - .append(ikey) - .append("|") - .append(matcherMac) - .append("|") - .append(destSubnetGatewayMac) - .append("|") - .append(srcEpFwdCtxOrds.getL3Id()) - .append("|") - .append(nextHop) - .toString()); MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType)) .setLayer3Match(m); addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id()))); - + Match match = mb.build(); + FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL3", match); FlowBuilder flowb = base().setId(flowid) .setPriority(Integer.valueOf(132)) - .setMatch(mb.build()) + .setMatch(match) .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build()); return flowb.build(); } @@ -953,8 +1423,7 @@ public class DestinationMapper extends FlowTable { * which we can't do because of the backwards way endpoints were * "architected". */ - return ctx.getPolicyResolver() - .getTenant(endpoint.getTenant()) + return ctx.getTenant(endpoint.getTenant()) .getEndpointGroup(endpoint.getEndpointGroup()) .getNetworkDomain(); } @@ -962,10 +1431,6 @@ public class DestinationMapper extends FlowTable { private HashSet getSubnets(final TenantId tenantId) { - // if (subnetsByTenant.get(tenantId) != null) { - // return subnetsByTenant.get(tenantId); - // } - if (ctx.getDataBroker() == null) { return null; } @@ -978,18 +1443,20 @@ public class DestinationMapper extends FlowTable { } catch (Exception e) { LOG.error("Could not read Tenant {}", tenantId, e); return null; + } finally { + t.close(); } - HashSet subnets = new HashSet(); - if (!tenantInfo.isPresent()) { LOG.warn("Tenant {} not found", tenantId); return null; } - subnets.addAll(tenantInfo.get().getSubnet()); - // subnetsByTenant.put(tenantId, subnets); - return subnets; + ForwardingContext fwCtx = tenantInfo.get().getForwardingContext(); + if (fwCtx == null || fwCtx.getSubnet() == null) { + return new HashSet<>(); + } + return new HashSet<>(fwCtx.getSubnet()); } // Need a method to get subnets for EPs attached to the node locally @@ -1002,8 +1469,8 @@ public class DestinationMapper extends FlowTable { for (Endpoint endpoint : endpointsForNode) { HashSet subnets = getSubnets(endpoint.getTenant()); if (subnets == null) { - LOG.error("No local subnets."); - return null; + LOG.debug("No local subnets in tenant {} for EP {}.", endpoint.getTenant(), endpoint.getKey()); + continue; } NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint); for (Subnet subnet : subnets) { @@ -1015,24 +1482,6 @@ public class DestinationMapper extends FlowTable { return localSubnets; } - /** - * Reads data from datastore as synchronous call. - * - * @return {@link Optional#isPresent()} is {@code true} if reading was - * successful and data exists in datastore; {@link Optional#isPresent()} is - * {@code false} otherwise - */ - public static Optional readFromDs(LogicalDatastoreType store, InstanceIdentifier path, - ReadTransaction rTx) { - CheckedFuture, ReadFailedException> resultFuture = rTx.read(store, path); - try { - return resultFuture.checkedGet(); - } catch (ReadFailedException e) { - LOG.warn("Read failed from DS.", e); - return Optional.absent(); - } - } - static byte[] bytesFromHexString(String values) { String target = ""; if (values != null) {