X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=renderers%2Fofoverlay%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fgroupbasedpolicy%2Frenderer%2Fofoverlay%2Fflow%2FDestinationMapper.java;h=1e8fa2a28728e8b6057bb4252fb13decb2ac2928;hb=7b948a309cbb014f952e5f14a251773a5d846c57;hp=039bd4259717e867ec7eecd3a256ff0474d33b50;hpb=3cb448f6323cd1a413dc5943ee0071d79e17ca81;p=groupbasedpolicy.git diff --git a/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java b/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java old mode 100644 new mode 100755 index 039bd4259..1e8fa2a28 --- a/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java +++ b/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved. + * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved. * * This program and the accompanying materials are made available under the * terms of the Eclipse Public License v1.0 which accompanies this distribution, @@ -8,12 +8,12 @@ package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow; +import static com.google.common.base.Preconditions.checkNotNull; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns; -import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.createNodePath; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum; @@ -25,13 +25,14 @@ import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtil import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction; -import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveRegTunIdAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction; +import static org.opendaylight.groupbasedpolicy.util.DataStoreHelper.readFromDs; import java.math.BigInteger; import java.util.ArrayList; @@ -46,93 +47,176 @@ import java.util.Objects; import java.util.Set; import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction; -import org.opendaylight.controller.md.sal.binding.api.ReadTransaction; import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType; -import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; -import org.opendaylight.groupbasedpolicy.endpoint.EpKey; +import org.opendaylight.groupbasedpolicy.dto.EgKey; +import org.opendaylight.groupbasedpolicy.dto.EpKey; +import org.opendaylight.groupbasedpolicy.dto.IndexedTenant; import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext; -import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.PolicyManager.FlowMap; +import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfWriter; +import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.endpoint.EndpointManager; import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch; import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals; -import org.opendaylight.groupbasedpolicy.resolver.EgKey; -import org.opendaylight.groupbasedpolicy.resolver.PolicyInfo; -import org.opendaylight.groupbasedpolicy.resolver.TenantUtils; +import org.opendaylight.groupbasedpolicy.util.IidFactory; +import org.opendaylight.groupbasedpolicy.util.TenantUtils; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress; import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action; -import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.go.to.table._case.GoToTable; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder; -import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.SubnetId; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.l3.prefix.fields.EndpointL3Gateways; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key; -import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.EndpointLocation.LocationType; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Prefix; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant; -import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.L3Context; -import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.Subnet; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.ForwardingContext; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L3Context; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.Subnet; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId; -import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.ethernet.match.fields.EthernetDestinationBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatchBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.Layer3Match; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder; -import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg0; +import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.dec.nw.ttl._case.DecNwTtl; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7; -import org.opendaylight.yangtools.yang.binding.DataObject; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Optional; +import com.google.common.base.Strings; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; -import com.google.common.util.concurrent.CheckedFuture; /** - * Manage the table that maps the destination address to the next hop for the - * path as well as applies any relevant routing transformations. + *

Manage the table that maps the destination address to the next hop for the + * path as well as applies any relevant routing transformations (table=3)

* + * Sync Ep flows, every endpoint pair creates L2 and L3 flow
+ * + * Also applies to L3 + *

+ * L2 Flows: + *

+ * External, local and remote L2 flows
+ * Priority = 50
+ * Matches:
+ * - dl_dst mac address {@link MacAddress}
+ * - loadReg4 {@link NxmNxReg4}
+ * Actions:
+ * - load tunnel Ipv4 (local and remote only)
+ * - loadReg2 {@link NxmNxReg2}
+ * - loadReg3 {@link NxmNxReg3}
+ * - loadReg7 (next hop) {@link NxmNxReg7}
+ * - {@link GoToTable} POLICY ENFORCER table
+ *

+ * L3 flows: + *

+ * External, local and remote L3 routed flows:
+ * Priority = 50
+ * Matches:
+ * - ip (ethertype) + * - dl_dst mac address {@link MacAddress}
+ * - setReg6 {@link NxmNxReg6}
+ * Actions:
+ * - loadReg2 {@link NxmNxReg2}
+ * - loadReg3 {@link NxmNxReg3}
+ * - loadReg4 (tunnel destination) {@link NxmNxReg4} (remote only)
+ * - loadReg7 (next hop) {@link NxmNxReg7}
+ * - set dst mac to eth_dst {@link MacAddress}
+ * - dec_ttl {@link DecNwTtl} (local only)
+ * - {@link GoToTable} POLICY ENFORCER table + *

+ * If virtual router ip is present in subnet, and subnet contains L3 context, arp flow is created
+ *

+ * Router Arp flow
+ * Priority = 150
+ * Matches:
+ * - arp (ethertype)
+ * - arp target transport address
+ * - setReg6 {@link NxmNxReg6}
+ * Actions:
+ * - move eth_src = eth_dst
+ * - set dl_src {@link MacAddress}
+ * - load arp_op
+ * - move arp_sha = arp_tha
+ * - load arp_sha
+ * - move arp_spa = arp_tpa
+ * - load arp_spa
+ * - output:port {@link NodeConnectorId}
+ *

+ * Broadcast flow (per flood domain) + * Priority = 140
+ * Matches:
+ * - ethernet destination {@link MacAddress} + * - setReg5 {@link NxmNxReg5}
+ * Actions:
+ * - load tunnel ID
+ * - group action
+ *

+ * L3 Prefix flow
+ * Priority = 140
+ * Matches:
+ * - ethernet destination {@link MacAddress} + * - setReg5 {@link NxmNxReg5}
+ * Actions:
+ * - dl_dst {@link MacAddress}
+ * - dec_ttl
+ * - loadReg2 {@link NxmNxReg2}
+ * - loadReg3 {@link NxmNxReg3}
+ * - loadReg4 (next hop) {@link NxmNxReg4}
+ * - loadReg7 (if internal, port_num == {@link NodeConnectorId of L2 EP} ) {@link NxmNxReg7}
+ * - loadReg7 (if external, port_num = external port) {@link NxmNxReg7}
+ * - {@link GoToTable} POLICY ENFORCER table */ public class DestinationMapper extends FlowTable { - protected static final Logger LOG = - LoggerFactory.getLogger(DestinationMapper.class); + + protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class); // TODO Li alagalah: Improve UT coverage for this class. // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows // This ensures we have the appropriate network-containment' - public static final short TABLE_ID = 2; + public static short TABLE_ID; /** * This is the MAC address of the magical router in the sky */ - public static final MacAddress ROUTER_MAC = - new MacAddress("88:f0:31:b5:12:b5"); - public static final MacAddress MULTICAST_MAC = - new MacAddress("01:00:00:00:00:00"); + public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5"); + public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00"); + public static final Integer BASE_L3_PRIORITY = 100; - public DestinationMapper(OfContext ctx) { + public DestinationMapper(OfContext ctx, short tableId) { super(ctx); + this.TABLE_ID = tableId; } Map> subnetsByTenant = new HashMap>(); @@ -143,108 +227,284 @@ public class DestinationMapper extends FlowTable { } @Override - public void sync(NodeId nodeId, PolicyInfo policyInfo, FlowMap flowMap) - throws Exception { - - TenantId currentTenant; - - flowMap.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null)); - - Set visitedEps = new HashSet<>(); - Set fdIds = new HashSet<>(); + public void sync(NodeId nodeId, OfWriter ofWriter) throws Exception { + + TenantId currentTenant = null; + + ofWriter.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null, TABLE_ID)); + + SetMultimap visitedEps = HashMultimap.create(); + Set epOrdSet = new HashSet<>(); + + for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) { + Set srcEpgIds = new HashSet<>(); + if (srcEp.getEndpointGroup() != null) + srcEpgIds.add(srcEp.getEndpointGroup()); + if (srcEp.getEndpointGroups() != null) + srcEpgIds.addAll(srcEp.getEndpointGroups()); + + for (EndpointGroupId epgId : srcEpgIds) { + EgKey epg = new EgKey(srcEp.getTenant(), epgId); + Set peers = Sets.union(Collections.singleton(epg), ctx.getCurrentPolicy().getPeers(epg)); + for (EgKey peer : peers) { + Collection endpointsForGroup = new HashSet<>(); + endpointsForGroup.addAll(ctx.getEndpointManager().getEndpointsForGroup(peer)); + endpointsForGroup.addAll(ctx.getEndpointManager().getExtEpsNoLocForGroup(peer)); + for (Endpoint peerEp : endpointsForGroup) { + currentTenant = peerEp.getTenant(); + subnetsByTenant.put(currentTenant, getSubnets(currentTenant)); + EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress()); + EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress()); + + if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) { + continue; + } + syncEP(ofWriter, nodeId, srcEp, peerEp); + visitedEps.put(srcEpKey, peerEpKey); + + // Process subnets and flood-domains for epPeer + EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, + peerEp); + if (epOrds == null) { + LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", peerEp); + continue; + } - for (EgKey epg : ctx.getEndpointManager().getGroupsForNode(nodeId)) { - Set peers = Sets.union(Collections.singleton(epg), - policyInfo.getPeers(epg)); - for (EgKey peer : peers) { - for (Endpoint epPeer : ctx.getEndpointManager().getEndpointsForGroup(peer)) { - currentTenant = epPeer.getTenant(); - subnetsByTenant.put(currentTenant, getSubnets(currentTenant)); - syncEP(flowMap, nodeId, policyInfo, epPeer, visitedEps); - // Process subnets and flood-domains for epPeer - EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, epPeer); - fdIds.add(epOrds.getFdId()); + epOrdSet.add(epOrds); + } } } } for (Entry> subnetEntry : subnetsByTenant.entrySet()) { - if(subnetEntry.getValue() == null ) { - LOG.trace("Tenant: {} has empty subnet entry.",subnetEntry.getKey()); + if (subnetEntry.getValue() == null) { + LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey()); continue; } - currentTenant=subnetEntry.getKey(); + currentTenant = subnetEntry.getKey(); for (Subnet sn : subnetEntry.getValue()) { - L3Context l3c = getL3ContextForSubnet(currentTenant,sn); + L3Context l3c = getL3ContextForSubnet(currentTenant, sn); Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn, OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId())); if (arpFlow != null) { - flowMap.writeFlow(nodeId, TABLE_ID, arpFlow); + ofWriter.writeFlow(nodeId, TABLE_ID, arpFlow); } else { - LOG.debug("ARP flow is not created, because virtual router IP has not been set for subnet {} .", + LOG.debug( + "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .", sn.getIpPrefix().getValue()); } } } // Write broadcast flows per flood domain. - for (Integer fdId : fdIds) { - if (groupExists(nodeId, fdId)) { - flowMap.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(fdId)); + for (EndpointFwdCtxOrdinals epOrd : epOrdSet) { + if (ofWriter.groupExists(nodeId, Integer.valueOf(epOrd.getFdId()).longValue())) { + ofWriter.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd)); + } + } + + // L3 Prefix Endpoint handling + Collection prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(currentTenant); + if (prefixEps != null) { + LOG.trace("DestinationMapper - Processing L3PrefixEndpoints"); + for (EndpointL3Prefix prefixEp : prefixEps) { + List localSubnets = getLocalSubnets(nodeId); + if (localSubnets == null) { + continue; + } + for (Subnet localSubnet: localSubnets) { + Flow prefixFlow = createL3PrefixFlow(prefixEp, nodeId, localSubnet); + if (prefixFlow != null) { + ofWriter.writeFlow(nodeId, TABLE_ID, prefixFlow); + LOG.trace("Wrote L3Prefix flow"); + } + } } } } + + // set up next-hop destinations for all the endpoints in the endpoint // group on the node - private Flow createBroadcastFlow(int fdId) { - FlowId flowId = new FlowId(new StringBuilder() - .append("broadcast|") - .append(fdId).toString()); - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(new EthernetMatchBuilder() - .setEthernetDestination(new EthernetDestinationBuilder() - .setAddress(MULTICAST_MAC) - .setMask(MULTICAST_MAC) - .build()) - .build()); - addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(fdId))); + private Flow createL3PrefixFlow(EndpointL3Prefix prefixEp, NodeId nodeId, Subnet subnet) throws Exception { + /* + * Priority: 100+lengthprefix + * Match: prefix, l3c, "mac address of router" ? + * Action: + * - set Reg2, Reg3 for L3Ep by L2Ep ? + * - if external, + * - Reg7: use switch location external port else punt for now + * - if internal + * - Reg7: grab L2Ep from L3Ep and use its location info + * - goto_table: POLENF (will check there for external on EP) + */ - FlowBuilder flowb = base() - .setPriority(Integer.valueOf(140)) - .setId(flowId) - .setMatch(mb.build()) - .setInstructions(instructions(applyActionIns(nxMoveRegTunIdAction(NxmNxReg0.class, false), - groupAction(Long.valueOf(fdId))))); - return flowb.build(); - } + ReadOnlyTransaction rTx = ctx.getDataBroker().newReadOnlyTransaction(); + // TODO Bug #3440 Target: Be - should support for more than first gateway. + EndpointL3Gateways l3Gateway = prefixEp.getEndpointL3Gateways().get(0); + Optional optL3Ep = readFromDs(LogicalDatastoreType.OPERATIONAL, + IidFactory.l3EndpointIid(l3Gateway.getL3Context(), l3Gateway.getIpAddress()), rTx); + if (!optL3Ep.isPresent()) { + LOG.error("createL3PrefixFlow - L3Endpoint gateway {} for L3Prefix {} not found.", l3Gateway, prefixEp); + return null; + } + EndpointL3 l3Ep = optL3Ep.get(); + if (l3Ep.getL2Context() == null || l3Ep.getMacAddress() == null) { + LOG.debug("L3 endpoint representing L3 gateway does not contain L2-context or MAC address. {}", l3Ep); + return null; + } + Optional optL2Ep = readFromDs(LogicalDatastoreType.OPERATIONAL, + IidFactory.endpointIid(l3Ep.getL2Context(), l3Ep.getMacAddress()), rTx); + if (!optL2Ep.isPresent()) { + LOG.error("createL3PrefixFlow - L2Endpoint for L3Gateway {} not found.", l3Ep); + return null; + } + Endpoint l2Ep = optL2Ep.get(); + EndpointFwdCtxOrdinals epFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, l2Ep); + if (epFwdCtxOrds == null) { + LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", l2Ep); + return null; + } - private boolean groupExists(NodeId nodeId, Integer fdId) throws Exception { - // Fetch existing GroupTables - if (ctx.getDataBroker() == null) { - return false; + NetworkDomainId epNetworkContainment = getEPNetworkContainment(l2Ep); + + MacAddress epDestMac = l2Ep.getMacAddress(); + MacAddress destSubnetGatewayMac = l2Ep.getMacAddress(); + L3Context destL3c = getL3ContextForSubnet(prefixEp.getTenant(), subnet); + if (destL3c == null || destL3c.getId() == null) { + LOG.error("No L3 Context found associated with subnet {}", subnet.getId()); + return null; } - ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction(); - InstanceIdentifier niid = createNodePath(nodeId); - Optional r = - t.read(LogicalDatastoreType.CONFIGURATION, niid).get(); - if (!r.isPresent()) - return false; - FlowCapableNode fcn = r.get().getAugmentation(FlowCapableNode.class); - if (fcn == null) - return false; - - if (fcn.getGroup() != null) { - for (Group g : fcn.getGroup()) { - if (g.getGroupId().getValue().equals(Long.valueOf(fdId))) { // Group - // Exists. - return true; + MacAddress matcherMac = routerPortMac(destL3c, subnet.getVirtualRouterIp()); + + ArrayList l3instructions = new ArrayList<>(); + List applyActions = new ArrayList<>(); + List l3ApplyActions = new ArrayList<>(); + + int order = 0; + + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setNextHop; + String nextHop=null; + + OfOverlayContext ofc = l2Ep.getAugmentation(OfOverlayContext.class); + + long portNum = -1; + if (EndpointManager.isInternal(l2Ep, ctx.getTenant(l2Ep.getTenant()).getExternalImplicitGroups())) { + checkNotNull(ofc.getNodeConnectorId()); + nextHop = ofc.getNodeConnectorId().getValue(); + try { + portNum = getOfPortNum(ofc.getNodeConnectorId()); + } catch (NumberFormatException ex) { + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); + return null; + } + + } else { + // External + Set externalPorts = ctx.getSwitchManager().getExternalPorts(nodeId); + checkNotNull(externalPorts); + for (NodeConnectorId externalPort : externalPorts) { + // TODO Bug #3440 Target: Be - should support for more than first external port. + //TODO Bug 3546 - Difficult: External port is unrelated to Tenant, L3C, L2BD.. + nextHop = externalPort.getValue(); + try { + portNum = getOfPortNum(externalPort); + } catch (NumberFormatException ex) { + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); + return null; } + continue; } } - return false; + + if (Strings.isNullOrEmpty(nextHop) + || portNum == -1) { + LOG.error("createL3Prefix - Cannot find nodeConnectorId for {} for Prefix: ", l2Ep, prefixEp); + return null; + } + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); + + Action setDlDst = setDlDstAction(epDestMac); + l3ApplyActions.add(setDlDst); + + Action decTtl = decNwTtlAction(); + l3ApplyActions.add(decTtl); + + order += 1; + applyActions.add(setdEPG); + applyActions.add(setdCG); + applyActions.add(setNextHop); + + applyActions.addAll(l3ApplyActions); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); + + l3instructions.add(applyActionsIns); + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER())) + .build(); + l3instructions.add(gotoTable); + + Layer3Match m = null; + Long etherType = null; + String ikey = null; + Integer prefixLength=0; + if (prefixEp.getIpPrefix().getIpv4Prefix() != null) { + ikey = prefixEp.getIpPrefix().getIpv4Prefix().getValue(); + etherType = IPv4; + prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv4Prefix().getValue().split("/")[1]); + m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build(); + } else if (prefixEp.getIpPrefix().getIpv6Prefix() != null) { + ikey = prefixEp.getIpPrefix().getIpv6Prefix().getValue(); + etherType = IPv6; + /* + * This will result in flows with priority between 100-228, but since its matching on IPv6 prefix as well + * this shouldn't pose and issue, as the priority is more important within the address space of the matcher, + * even though technically flows are processed in priority order. + */ + + prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv6Prefix().getValue().split("/")[1]); + m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build(); + } else { + LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", prefixEp); + return null; + } + + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType)); + addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id()))); + Match match = mb.build(); + FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "L3prefix", match); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(BASE_L3_PRIORITY+prefixLength)) + .setMatch(match) + .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build()); + return flowb.build(); + } + + private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) { + MatchBuilder mb = new MatchBuilder() + .setEthernetMatch(new EthernetMatchBuilder().setEthernetDestination( + new EthernetDestinationBuilder().setAddress(MULTICAST_MAC) + .setMask(MULTICAST_MAC) + .build()).build()); + addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId()))); + + Match match = mb.build(); + FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "broadcast", match); + FlowBuilder flowb = base().setPriority(Integer.valueOf(140)) + .setId(flowId) + .setMatch(match) + .setInstructions( + instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false), + groupAction(Long.valueOf(epOrd.getFdId()))))); + + return flowb.build(); } private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) { @@ -256,9 +516,9 @@ public class DestinationMapper extends FlowTable { MacAddress defaultMacAddress = ROUTER_MAC; EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId()); - InstanceIdentifier endpointsIid = - InstanceIdentifier.builder(Endpoints.class) - .child(EndpointL3.class, l3Key).build(); + InstanceIdentifier endpointsIid = InstanceIdentifier.builder(Endpoints.class) + .child(EndpointL3.class, l3Key) + .build(); ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction(); Optional r; @@ -279,7 +539,12 @@ public class DestinationMapper extends FlowTable { } private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) { - L3Context l3c = ctx.getPolicyResolver().getTenant(tenantId).resolveL3Context(sn.getId()); + IndexedTenant indexedTenant = ctx.getTenant(tenantId); + if (indexedTenant == null) { + LOG.debug("Tenant {} is null, cannot get L3 context", tenantId); + return null; + } + L3Context l3c = indexedTenant.resolveL3Context(sn.getId()); return l3c; } @@ -308,39 +573,27 @@ public class DestinationMapper extends FlowTable { BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue())); - FlowId flowId = new FlowId(new StringBuffer() - .append("routerarp|") - .append(sn.getId().getValue()) - .append("|") - .append(ikey) - .append("|") - .append(l3Id) - .toString()); - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(ethernetMatch(null, null, ARP)) - .setLayer3Match(new ArpMatchBuilder() - .setArpOp(Integer.valueOf(1)) - .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32")) - .build()); - addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, - Long.valueOf(l3Id))); - - FlowBuilder flowb = base() - .setPriority(150) - .setId(flowId) - .setMatch(mb.build()) - .setInstructions(instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), - setDlSrcAction(routerMac), - nxLoadArpOpAction(BigInteger.valueOf(2L)), - nxMoveArpShaToArpThaAction(), - nxLoadArpShaAction(intRouterMac), - nxMoveArpSpaToArpTpaAction(), - nxLoadArpSpaAction(ikey), - outputAction(new NodeConnectorId(nodeId.getValue() + ":INPORT"))))); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match( + new ArpMatchBuilder().setArpOp(Integer.valueOf(1)) + .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32")) + .build()); + addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id))); + + Match match = mb.build(); + FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "routerarp", match); + FlowBuilder flowb = base().setPriority(150) + .setId(flowId) + .setMatch(match) + .setInstructions( + instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac), + nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(), + nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(), + nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue() + + ":INPORT"))))); return flowb.build(); } else { - LOG.warn("IPv6 virtual router {} for subnet {} not supported", - sn.getVirtualRouterIp(), sn.getId().getValue()); + LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId() + .getValue()); return null; } @@ -354,10 +607,8 @@ public class DestinationMapper extends FlowTable { int order = 0; - Action setdEPG = nxLoadRegAction(NxmNxReg2.class, - BigInteger.valueOf(epFwdCtxOrds.getEpgId())); - Action setdCG = nxLoadRegAction(NxmNxReg3.class, - BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId())); Action setNextHop; String nextHop; @@ -368,13 +619,11 @@ public class DestinationMapper extends FlowTable { try { portNum = getOfPortNum(ofc.getNodeConnectorId()); } catch (NumberFormatException ex) { - LOG.warn("Could not parse port number {}", - ofc.getNodeConnectorId(), ex); + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); return null; } - setNextHop = nxLoadRegAction(NxmNxReg7.class, - BigInteger.valueOf(portNum)); + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); // END L2 LOCAL @@ -382,70 +631,56 @@ public class DestinationMapper extends FlowTable { applyActions.add(setdEPG); applyActions.add(setdCG); applyActions.add(setNextHop); - Instruction applyActionsIns = new InstructionBuilder() - .setOrder(order++) - .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) - .build(); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); instructions.add(applyActionsIns); - Instruction gotoTable = new InstructionBuilder() - .setOrder(order++) - .setInstruction(gotoTableIns((short) (getTableId() + 1))) - .build(); + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER())) + .build(); instructions.add(gotoTable); - FlowId flowid = new FlowId(new StringBuilder() - .append(epFwdCtxOrds.getBdId()) - .append("|l2|") - .append(ep.getMacAddress().getValue()) - .append("|") - .append(nextHop) - .toString()); - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(ethernetMatch(null, - ep.getMacAddress(), - null)); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null)); addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId()))); - FlowBuilder flowb = base() - .setId(flowid) - .setPriority(Integer.valueOf(50)) - .setMatch(mb.build()) - .setInstructions(new InstructionsBuilder() - .setInstruction(instructions) - .build()); + Match match = mb.build(); + FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL2", match); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(50)) + .setMatch(match) + .setInstructions(new InstructionsBuilder().setInstruction(instructions).build()); return flowb.build(); } - private void syncEP(FlowMap flowMap, - NodeId nodeId, PolicyInfo policyInfo, - Endpoint epPeer, Set visitedEps) + private void syncEP(OfWriter ofWriter, NodeId nodeId, Endpoint srcEp, Endpoint destEp) throws Exception { - EpKey epPeerKey = new EpKey(epPeer.getL2Context(), epPeer.getMacAddress()); - - if (visitedEps.contains(epPeerKey)) { + if (ctx.getTenant(srcEp.getTenant()) == null + || ctx.getTenant(destEp.getTenant()) == null) { + LOG.debug("Source or destination EP references empty tenant srcEp:{} destEp:{}", srcEp, destEp); return; } - visitedEps.add(epPeerKey); // TODO: Conditions messed up, but for now, send policyInfo until this // is fixed. - EndpointFwdCtxOrdinals epFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, epPeer); - - if (epPeer.getTenant() == null || (epPeer.getEndpointGroup() == null && epPeer.getEndpointGroups() == null)) { - LOG.trace("Didn't process endpoint due to either tenant, or EPG(s) being null", epPeer.getKey()); + EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, destEp); + if (destEpFwdCtxOrds == null) { + LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", destEp); + return; + } + EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, srcEp); + if (srcEpFwdCtxOrds == null) { + LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", srcEp); return; } - OfOverlayContext ofc = epPeer.getAugmentation(OfOverlayContext.class); - // //////////////////////////////////////////////////////////////////////////////////////// - /* - * NOT HANDLING EXTERNALS - */ - if (LocationType.External.equals(ofc.getLocationType())) { - // XXX - TODO - perform NAT and send to the external network - // TODO: Use case neutron gateway interface - LOG.warn("External endpoints not yet supported"); + + if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) { + if (destEp.getTenant() == null) { + LOG.debug("Didn't process endpoint {} due to tenant being null", destEp.getKey()); + } else { + LOG.debug("Didn't process endpoint {} due to EPG(s) being null", destEp.getKey()); + } return; } @@ -461,29 +696,65 @@ public class DestinationMapper extends FlowTable { return; } - if (Objects.equals(ofc.getNodeId(), nodeId)) { + OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class); + + // forwarding outside of internal domain should be done when dest EP or GW is external. + Subnet srcSubnet = ctx.getTenant(srcEp.getTenant()).resolveSubnet(new SubnetId(srcEp.getNetworkContainment())); + Endpoint l2Gw = this.getL2EndpointOfSubnetGateway(srcEp.getTenant(), srcSubnet); + boolean destEpIsExternal = destEp.getNetworkContainment() != null + && EndpointManager.isExternal(destEp, ctx.getTenant(destEp.getTenant()).getExternalImplicitGroups()); + boolean subnetGwIsExternal = l2Gw != null + && EndpointManager.isExternal(l2Gw, ctx.getTenant(l2Gw.getTenant()).getExternalImplicitGroups()); + if (destEpIsExternal || subnetGwIsExternal) { + if (ofc == null && destEp.getNetworkContainment().equals(srcEp.getNetworkContainment())) { + Flow flow = createExternalL2Flow(destEp, destEpFwdCtxOrds, nodeId); + if (flow != null) { + ofWriter.writeFlow(nodeId, TABLE_ID, flow); + } + } else if (l2Gw != null && EndpointManager.isExternal(l2Gw, ctx.getTenant(l2Gw.getTenant()).getExternalImplicitGroups()) + && !destEp.getNetworkContainment().equals(srcEp.getNetworkContainment())) { + for (L3Address l3a : destEp.getL3Address()) { + if (l3a.getIpAddress() == null || l3a.getL3Context() == null) { + LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}", + destEp.getL3Address()); + continue; + } + for (Subnet localSubnet : localSubnets) { + Flow extL3Flow = createExternalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, localSubnet, nodeId); + if (extL3Flow != null) { + ofWriter.writeFlow(nodeId, TABLE_ID, extL3Flow); + } else { + LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(), + localSubnet.getIpPrefix().getValue()); + } + } + } + } + } + else if (ofc != null && Objects.equals(ofc.getNodeId(), nodeId)) { // this is a local endpoint; send to the approppriate local // port - flowMap.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(epPeer, epFwdCtxOrds, ofc)); - + if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) { + ofWriter.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc)); + } // TODO Li alagalah: Need to move to EndpointL3 for L3 processing. // The Endpoint conflation must end! - if (epPeer.getL3Address() == null) { - LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", epPeer.getKey()); + if (destEp.getL3Address() == null) { + LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey()); return; } - for (L3Address l3a : epPeer.getL3Address()) { + for (L3Address l3a : destEp.getL3Address()) { if (l3a.getIpAddress() == null || l3a.getL3Context() == null) { LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}", - epPeer.getL3Address()); + destEp.getL3Address()); continue; } else { for (Subnet localSubnet : localSubnets) { - Flow flow = createLocalL3Flow(epPeer, l3a, epFwdCtxOrds, ofc, localSubnet); + Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet); if (flow != null) { - flowMap.writeFlow(nodeId, TABLE_ID, flow); + ofWriter.writeFlow(nodeId, TABLE_ID, flow); } else { LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(), localSubnet.getIpPrefix().getValue()); @@ -491,31 +762,36 @@ public class DestinationMapper extends FlowTable { } } } - } else { + } else if(ofc!= null) { // this endpoint is on a different switch; send to the // appropriate tunnel - - Flow remoteL2Flow = createRemoteL2Flow(epPeer, nodeId, epFwdCtxOrds, ofc); - if (remoteL2Flow != null) { - flowMap.writeFlow(nodeId, TABLE_ID, remoteL2Flow); + if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) { + Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc); + if (remoteL2Flow != null) { + ofWriter.writeFlow(nodeId, TABLE_ID, remoteL2Flow); + } + } else { + LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}", + srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId()); } // TODO Li alagalah: Need to move to EndpointL3 for L3 processing. // The Endpoint conflation must end! - if (epPeer.getL3Address() == null) { - LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", epPeer.getKey()); + if (destEp.getL3Address() == null) { + LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey()); return; } - for (L3Address l3a : epPeer.getL3Address()) { + for (L3Address l3a : destEp.getL3Address()) { if (l3a.getIpAddress() == null || l3a.getL3Context() == null) { LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}", - epPeer.getL3Address()); + destEp.getL3Address()); continue; } else { for (Subnet localSubnet : localSubnets) { - Flow remoteL3Flow = createRemoteL3Flow(epPeer, l3a, nodeId, epFwdCtxOrds, ofc, localSubnet); + Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds, + destEpFwdCtxOrds, ofc, localSubnet); if (remoteL3Flow != null) { - flowMap.writeFlow(nodeId, TABLE_ID, remoteL3Flow); + ofWriter.writeFlow(nodeId, TABLE_ID, remoteL3Flow); } else { LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(), localSubnet.getIpPrefix().getValue()); @@ -533,7 +809,7 @@ public class DestinationMapper extends FlowTable { * ################################## DestMapper Flow methods * ################################## */ - private Flow createLocalL3Flow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds, + private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc, Subnet srcSubnet) { // TODO Li alagalah - refactor common code but keep simple method @@ -541,10 +817,10 @@ public class DestinationMapper extends FlowTable { Subnet destSubnet = null; HashSet subnets = getSubnets(destEp.getTenant()); if (subnets == null) { - LOG.trace("No subnets in tenant {}", destL3Address.getIpAddress()); + LOG.trace("No subnets in tenant {}", destEp.getTenant()); return null; } - NetworkDomainId epNetworkContainment= getEPNetworkContainment(destEp); + NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp); for (Subnet subnet : subnets) { // TODO Li alagalah add IPv6 support if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) { @@ -580,19 +856,17 @@ public class DestinationMapper extends FlowTable { if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) { LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId() - .getValue(), destL3c.getId().getValue()); + .getValue(), destL3c.getId().getValue()); return null; } - MacAddress destMacAddress = routerPortMac(destL3c, destSubnet.getVirtualRouterIp()); - MacAddress srcMacAddress = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp()); + MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp()); + MacAddress epDestMac = destEp.getMacAddress(); + MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp()); - // TODO Li alagalah: using gateway field instead of overloading - // virtual-router-ip will remove need for the && check. - if (srcMacAddress.getValue().equals(destMacAddress.getValue()) && - !(srcMacAddress.getValue().equals(ROUTER_MAC.getValue()))) { - LOG.trace("Same source and destination MacAddress in createL3 Flow, this flow not needed as handled by ARP"); - return null; + if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) { + // This is our final destination, so match on actual EP mac. + matcherMac = epDestMac; } ArrayList l3instructions = new ArrayList<>(); @@ -601,16 +875,11 @@ public class DestinationMapper extends FlowTable { int order = 0; - Action setdEPG = nxLoadRegAction(NxmNxReg2.class, - BigInteger.valueOf(epFwdCtxOrds.getEpgId())); - Action setdCG = nxLoadRegAction(NxmNxReg3.class, - BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId())); Action setNextHop; String nextHop; - Action setDlSrc = setDlSrcAction(destMacAddress); - Action decTtl = decNwTtlAction(); - // BEGIN L3 LOCAL nextHop = ofc.getNodeConnectorId().getValue(); @@ -618,36 +887,175 @@ public class DestinationMapper extends FlowTable { try { portNum = getOfPortNum(ofc.getNodeConnectorId()); } catch (NumberFormatException ex) { - LOG.warn("Could not parse port number {}", - ofc.getNodeConnectorId(), ex); + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); return null; } - setNextHop = nxLoadRegAction(NxmNxReg7.class, - BigInteger.valueOf(portNum)); + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); + // END L3 LOCAL - Action setDlDst = setDlDstAction(destEp.getMacAddress()); + // Lets not re-write the srcMac if its local. + if (!(matcherMac.getValue().equals(epDestMac.getValue()))) { + Action setDlSrc = setDlSrcAction(destSubnetGatewayMac); + l3ApplyActions.add(setDlSrc); + } + + Action setDlDst = setDlDstAction(epDestMac); l3ApplyActions.add(setDlDst); + + Action decTtl = decNwTtlAction(); + l3ApplyActions.add(decTtl); + + order += 1; + applyActions.add(setdEPG); + applyActions.add(setdCG); + applyActions.add(setNextHop); + + applyActions.addAll(l3ApplyActions); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); + + l3instructions.add(applyActionsIns); + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER())) + .build(); + l3instructions.add(gotoTable); + Layer3Match m = null; + Long etherType = null; + String ikey = null; + if (destL3Address.getIpAddress().getIpv4Address() != null) { + ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32"; + etherType = IPv4; + m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build(); + } else if (destL3Address.getIpAddress().getIpv6Address() != null) { + ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128"; + etherType = IPv6; + m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build(); + } else { + LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString()); + return null; + } + + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType)) + .setLayer3Match(m); + addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id()))); + Match match = mb.build(); + FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL3", match); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(132)) + .setMatch(match) + .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build()); + return flowb.build(); + } + + private Flow createExternalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds, + Subnet srcSubnet, NodeId nodeId) { + + Subnet destSubnet = null; + HashSet subnets = getSubnets(destEp.getTenant()); + if (subnets == null) { + LOG.trace("No subnets in tenant {}", destEp.getTenant()); + return null; + } + NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp); + for (Subnet subnet : subnets) { + // TODO Li alagalah add IPv6 support + if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) { + destSubnet = subnet; + break; + } + } + if (destSubnet == null) { + LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress()); + return null; + } + + if (destSubnet.getVirtualRouterIp() == null) { + LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(), + destL3Address.getKey()); + return null; + } + + if (srcSubnet.getVirtualRouterIp() == null) { + LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix()); + return null; + } + L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet); + if (destL3c == null || destL3c.getId() == null) { + LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId()); + return null; + } + L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet); + if (srcL3c == null || srcL3c.getId() == null) { + LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId()); + return null; + } + + if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) { + LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId() + .getValue(), destL3c.getId().getValue()); + return null; + } + + Endpoint l2Gw = getL2EndpointOfSubnetGateway(destEp.getTenant(), srcSubnet); + if(l2Gw == null) { + LOG.warn("The endpoint representing external gateway of subnet {} not found", srcSubnet); + return null; + } + MacAddress matcherMac = destEp.getMacAddress(); + MacAddress destSubnetGatewayMac = l2Gw.getMacAddress(); + + ArrayList l3instructions = new ArrayList<>(); + List applyActions = new ArrayList<>(); + List l3ApplyActions = new ArrayList<>(); + + int order = 0; + + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setNextHop; + + Set extPorts = ctx.getSwitchManager().getExternalPorts(nodeId); + if (extPorts == null || !extPorts.iterator().hasNext()) { + LOG.warn("No external interface on node: {}. External Gateway {} is not reachable!", nodeId, l2Gw.getKey()); + return null; + } + // only one external port is supported for now + NodeConnectorId extPort = extPorts.iterator().next(); + + long portNum; + try { + portNum = getOfPortNum(extPort); + } catch (NumberFormatException ex) { + LOG.warn("Could not parse port number {}", extPort, ex); + return null; + } + + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); // END L3 LOCAL + + Action setDlSrc = setDlSrcAction(destSubnetGatewayMac); l3ApplyActions.add(setDlSrc); - l3ApplyActions.add(decTtl); + + Action setDlDst = setDlDstAction(l2Gw.getMacAddress()); + l3ApplyActions.add(setDlDst); + order += 1; applyActions.add(setdEPG); applyActions.add(setdCG); applyActions.add(setNextHop); applyActions.addAll(l3ApplyActions); - Instruction applyActionsIns = new InstructionBuilder() - .setOrder(order++) - .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) - .build(); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); l3instructions.add(applyActionsIns); - Instruction gotoTable = new InstructionBuilder() - .setOrder(order++) - .setInstruction(gotoTableIns((short) (getTableId() + 1))) - .build(); + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER())) + .build(); l3instructions.add(gotoTable); Layer3Match m = null; Long etherType = null; @@ -655,54 +1063,50 @@ public class DestinationMapper extends FlowTable { if (destL3Address.getIpAddress().getIpv4Address() != null) { ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32"; etherType = IPv4; - m = new Ipv4MatchBuilder() - .setIpv4Destination(new Ipv4Prefix(ikey)) - .build(); + m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build(); } else if (destL3Address.getIpAddress().getIpv6Address() != null) { ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128"; etherType = IPv6; - m = new Ipv6MatchBuilder() - .setIpv6Destination(new Ipv6Prefix(ikey)) - .build(); + m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build(); } else { LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString()); return null; } - FlowId flowid = new FlowId(new StringBuilder() - .append(Integer.toString(epFwdCtxOrds.getL3Id())) - .append("|l3|") - .append(ikey) - .append("|") - .append(Integer.toString(epFwdCtxOrds.getEpgId())) - .append("|") - .append(Integer.toString(epFwdCtxOrds.getCgId())) - .append("|") - .append(srcMacAddress) - .append("|") - .append(destMacAddress) - .append("|") - .append(nextHop) - .toString()); - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(ethernetMatch(null, - srcMacAddress, - etherType)) - .setLayer3Match(m); - addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, - Long.valueOf(epFwdCtxOrds.getL3Id()))); - FlowBuilder flowb = base() - .setId(flowid) - .setPriority(Integer.valueOf(132)) - .setMatch(mb.build()) - .setInstructions(new InstructionsBuilder() - .setInstruction(l3instructions) - .build()); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType)) + .setLayer3Match(m); + addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id()))); + Match match = mb.build(); + FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "externalL3", match); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(132)) + .setMatch(match) + .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build()); return flowb.build(); } - private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals epFwdCtxOrds, - OfOverlayContext ofc) { + private Endpoint getL2EndpointOfSubnetGateway(TenantId tenantId, Subnet subnet) { + if (subnet != null && subnet.getVirtualRouterIp() != null) { + IpAddress gwIpAddress = subnet.getVirtualRouterIp(); + Collection prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(tenantId); + if (prefixEps != null) { + for (EndpointL3Prefix prefixEp : prefixEps) { + for (EndpointL3Gateways gw : prefixEp.getEndpointL3Gateways()) { + EndpointL3 l3Ep = ctx.getEndpointManager().getL3Endpoint(gw.getL3Context(), gwIpAddress, + prefixEp.getTenant()); + if (l3Ep != null && l3Ep.getL2Context() != null && l3Ep.getMacAddress() != null) { + return ctx.getEndpointManager().getEndpoint( + new EpKey(l3Ep.getL2Context(), l3Ep.getMacAddress())); + } + } + } + } + } + return null; + } + + private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds, + EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) { // TODO Li alagalah - refactor common code but keep simple method @@ -714,18 +1118,14 @@ public class DestinationMapper extends FlowTable { int order = 0; - Action setdEPG = nxLoadRegAction(NxmNxReg2.class, - BigInteger.valueOf(epFwdCtxOrds.getEpgId())); - Action setdCG = nxLoadRegAction(NxmNxReg3.class, - BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId())); Action setNextHop; String nextHop; // BEGIN TUNNEL HANDLING - IpAddress tunDst = - ctx.getSwitchManager().getTunnelIP(ofc.getNodeId()); - NodeConnectorId tunPort = - ctx.getSwitchManager().getTunnelPort(nodeId); + IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class); + NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class); if (tunDst == null) { LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep); return null; @@ -742,8 +1142,7 @@ public class DestinationMapper extends FlowTable { tundstAction = nxLoadTunIPv4Action(nextHop, false); } else if (tunDst.getIpv6Address() != null) { // nextHop = tunDst.getIpv6Address().getValue(); - LOG.error("IPv6 tunnel destination {} for {} not supported", - tunDst.getIpv6Address().getValue(), + LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(), ofc.getNodeId()); return null; } else { @@ -756,17 +1155,11 @@ public class DestinationMapper extends FlowTable { try { portNum = getOfPortNum(tunPort); } catch (NumberFormatException ex) { - LOG.warn("Could not parse port number {}", - ofc.getNodeConnectorId(), ex); + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); return null; } - setNextHop = nxLoadRegAction(NxmNxReg7.class, - BigInteger.valueOf(portNum)); - Action tunIdAction = - nxMoveRegTunIdAction(NxmNxReg0.class, false); - - applyActions.add(tunIdAction); + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); applyActions.add(tundstAction); // END TUNNEL @@ -774,48 +1167,89 @@ public class DestinationMapper extends FlowTable { applyActions.add(setdEPG); applyActions.add(setdCG); applyActions.add(setNextHop); - Instruction applyActionsIns = new InstructionBuilder() - .setOrder(order++) - .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) - .build(); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); instructions.add(applyActionsIns); - applyActionsIns = new InstructionBuilder() - .setOrder(order++) - .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) - .build(); + applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); - Instruction gotoTable = new InstructionBuilder() - .setOrder(order++) - .setInstruction(gotoTableIns((short) (getTableId() + 1))) - .build(); + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER())) + .build(); instructions.add(gotoTable); - FlowId flowid = new FlowId(new StringBuilder() - .append(epFwdCtxOrds.getBdId()) - .append("|l2|") - .append(ep.getMacAddress().getValue()) - .append("|") - .append(nextHop) - .toString()); - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(ethernetMatch(null, - ep.getMacAddress(), - null)); - addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId()))); - FlowBuilder flowb = base() - .setId(flowid) - .setPriority(Integer.valueOf(50)) - .setMatch(mb.build()) - .setInstructions(new InstructionsBuilder() - .setInstruction(instructions) - .build()); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null)); + addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId()))); + Match match = mb.build(); + FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL2", match); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(50)) + .setMatch(match) + .setInstructions(new InstructionsBuilder().setInstruction(instructions).build()); return flowb.build(); } - private Flow createRemoteL3Flow(Endpoint destEp, L3Address destL3Address, NodeId nodeId, - EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc, Subnet srcSubnet) { + private Flow createExternalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds,NodeId nodeId) { + + ArrayList instructions = new ArrayList<>(); + List applyActions = new ArrayList<>(); + + int order = 0; + + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setNextHop; + + // BEGIN L2 LOCAL + Set extPorts = ctx.getSwitchManager().getExternalPorts(nodeId); + if(extPorts == null || !extPorts.iterator().hasNext()) { + return null; + } + // Only one external port is currently supported. + NodeConnectorId extPort = extPorts.iterator().next(); + long portNum; + try { + portNum = getOfPortNum(extPort); + } catch (NumberFormatException ex) { + LOG.warn("Could not parse port number {}", extPort, ex); + return null; + } + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); + + // END L2 LOCAL + + order += 1; + applyActions.add(setdEPG); + applyActions.add(setdCG); + applyActions.add(setNextHop); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); + instructions.add(applyActionsIns); + + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER())) + .build(); + instructions.add(gotoTable); + + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null)); + addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId()))); + Match match = mb.build(); + FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "externalL2", match); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(50)) + .setMatch(match) + .setInstructions(new InstructionsBuilder().setInstruction(instructions).build()); + return flowb.build(); + } + + private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId, + EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc, + Subnet srcSubnet) { // TODO Li alagalah - refactor common code but keep simple method @@ -824,10 +1258,10 @@ public class DestinationMapper extends FlowTable { Subnet destSubnet = null; HashSet subnets = getSubnets(destEp.getTenant()); if (subnets == null) { - LOG.trace("No subnets in tenant {}", destL3Address.getIpAddress()); + LOG.trace("No subnets in tenant {}", destEp.getTenant()); return null; } - NetworkDomainId epNetworkContainment= getEPNetworkContainment(destEp); + NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp); for (Subnet subnet : subnets) { // TODO Li alagalah add IPv6 support if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) { @@ -850,55 +1284,44 @@ public class DestinationMapper extends FlowTable { LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix()); return null; } - L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(),destSubnet); + L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet); if (destL3c == null || destL3c.getId() == null) { LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId()); return null; } - L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(),srcSubnet); + L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet); if (srcL3c == null || srcL3c.getId() == null) { LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId()); return null; } if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) { - LOG.info("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId() - .getValue(), destL3c.getId().getValue()); + LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId() + .getValue(), destL3c.getId().getValue()); return null; } - MacAddress destMacAddress = routerPortMac(destL3c, destSubnet.getVirtualRouterIp()); - MacAddress srcMacAddress = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp()); - - // TODO Li alagalah: using gateway field instead of overloading - // virtual-router-ip will remove need for the && check. - if (srcMacAddress.getValue().equals(destMacAddress.getValue()) && - !(srcMacAddress.getValue().equals(ROUTER_MAC.getValue()))) { - LOG.trace("Same source and destination MacAddress in createL3 Flow, this flow not needed as handled by ARP"); - return null; + MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp()); + MacAddress epDestMac = destEp.getMacAddress(); + MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp()); + if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) { + // This is our final destination, so match on actual EP mac. + matcherMac = epDestMac; } - ArrayList l3instructions = new ArrayList<>(); List applyActions = new ArrayList<>(); List l3ApplyActions = new ArrayList<>(); int order = 0; - Action setdEPG = nxLoadRegAction(NxmNxReg2.class, - BigInteger.valueOf(epFwdCtxOrds.getEpgId())); - Action setdCG = nxLoadRegAction(NxmNxReg3.class, - BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId())); Action setNextHop; String nextHop; - Action setDlSrc = setDlSrcAction(srcMacAddress); - Action decTtl = decNwTtlAction(); - // BEGIN TUNNEL HANDLING - IpAddress tunDst = - ctx.getSwitchManager().getTunnelIP(ofc.getNodeId()); - NodeConnectorId tunPort = - ctx.getSwitchManager().getTunnelPort(nodeId); + IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class); + NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class); if (tunDst == null) { LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address); return null; @@ -915,8 +1338,7 @@ public class DestinationMapper extends FlowTable { tundstAction = nxLoadTunIPv4Action(nextHop, false); } else if (tunDst.getIpv6Address() != null) { // nextHop = tunDst.getIpv6Address().getValue(); - LOG.error("IPv6 tunnel destination {} for {} not supported", - tunDst.getIpv6Address().getValue(), + LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(), ofc.getNodeId()); return null; } else { @@ -929,40 +1351,40 @@ public class DestinationMapper extends FlowTable { try { portNum = getOfPortNum(tunPort); } catch (NumberFormatException ex) { - LOG.warn("Could not parse port number {}", - ofc.getNodeConnectorId(), ex); + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); return null; } - setNextHop = nxLoadRegAction(NxmNxReg7.class, - BigInteger.valueOf(portNum)); - Action tunIdAction = - nxMoveRegTunIdAction(NxmNxReg0.class, false); - - applyActions.add(tunIdAction); + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); applyActions.add(tundstAction); // END TUNNEL - l3ApplyActions.add(setDlSrc); - l3ApplyActions.add(decTtl); order += 1; applyActions.add(setdEPG); applyActions.add(setdCG); applyActions.add(setNextHop); - l3ApplyActions.add(setDlSrc); + + // Lets not re-write the srcMac if its local. + if (!(matcherMac.getValue().equals(epDestMac.getValue()))) { + Action setDlSrc = setDlSrcAction(destSubnetGatewayMac); + l3ApplyActions.add(setDlSrc); + } + + Action setDlDst = setDlDstAction(epDestMac); + l3ApplyActions.add(setDlDst); + + Action decTtl = decNwTtlAction(); l3ApplyActions.add(decTtl); applyActions.addAll(l3ApplyActions); - Instruction applyActionsIns = new InstructionBuilder() - .setOrder(order++) - .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) - .build(); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); l3instructions.add(applyActionsIns); - Instruction gotoTable = new InstructionBuilder() - .setOrder(order++) - .setInstruction(gotoTableIns((short) (getTableId() + 1))) - .build(); + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER())) + .build(); l3instructions.add(gotoTable); Layer3Match m = null; Long etherType = null; @@ -970,49 +1392,25 @@ public class DestinationMapper extends FlowTable { if (destL3Address.getIpAddress().getIpv4Address() != null) { ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32"; etherType = IPv4; - m = new Ipv4MatchBuilder() - .setIpv4Destination(new Ipv4Prefix(ikey)) - .build(); + m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build(); } else if (destL3Address.getIpAddress().getIpv6Address() != null) { ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128"; etherType = IPv6; - m = new Ipv6MatchBuilder() - .setIpv6Destination(new Ipv6Prefix(ikey)) - .build(); + m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build(); } else { LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString()); return null; } - FlowId flowid = new FlowId(new StringBuilder() - .append(Integer.toString(epFwdCtxOrds.getL3Id())) - .append("|l3|") - .append(ikey) - .append("|") - .append(Integer.toString(epFwdCtxOrds.getEpgId())) - .append("|") - .append(Integer.toString(epFwdCtxOrds.getCgId())) - .append("|") - .append(srcMacAddress) - .append("|") - .append(destMacAddress) - .append("|") - .append(nextHop) - .toString()); - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(ethernetMatch(null, - destMacAddress, - etherType)) - .setLayer3Match(m); - addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, - Long.valueOf(epFwdCtxOrds.getL3Id()))); - FlowBuilder flowb = base() - .setId(flowid) - .setPriority(Integer.valueOf(132)) - .setMatch(mb.build()) - .setInstructions(new InstructionsBuilder() - .setInstruction(l3instructions) - .build()); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType)) + .setLayer3Match(m); + addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id()))); + Match match = mb.build(); + FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL3", match); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(132)) + .setMatch(match) + .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build()); return flowb.build(); } @@ -1025,17 +1423,14 @@ public class DestinationMapper extends FlowTable { * which we can't do because of the backwards way endpoints were * "architected". */ - return ctx.getPolicyResolver().getTenant(endpoint.getTenant()) - .getEndpointGroup(endpoint.getEndpointGroup()).getNetworkDomain(); + return ctx.getTenant(endpoint.getTenant()) + .getEndpointGroup(endpoint.getEndpointGroup()) + .getNetworkDomain(); } } private HashSet getSubnets(final TenantId tenantId) { - // if (subnetsByTenant.get(tenantId) != null) { - // return subnetsByTenant.get(tenantId); - // } - if (ctx.getDataBroker() == null) { return null; } @@ -1048,18 +1443,20 @@ public class DestinationMapper extends FlowTable { } catch (Exception e) { LOG.error("Could not read Tenant {}", tenantId, e); return null; + } finally { + t.close(); } - HashSet subnets = new HashSet(); - if (!tenantInfo.isPresent()) { LOG.warn("Tenant {} not found", tenantId); return null; } - subnets.addAll(tenantInfo.get().getSubnet()); - // subnetsByTenant.put(tenantId, subnets); - return subnets; + ForwardingContext fwCtx = tenantInfo.get().getForwardingContext(); + if (fwCtx == null || fwCtx.getSubnet() == null) { + return new HashSet<>(); + } + return new HashSet<>(fwCtx.getSubnet()); } // Need a method to get subnets for EPs attached to the node locally @@ -1072,10 +1469,10 @@ public class DestinationMapper extends FlowTable { for (Endpoint endpoint : endpointsForNode) { HashSet subnets = getSubnets(endpoint.getTenant()); if (subnets == null) { - LOG.error("No local subnets."); - return null; + LOG.debug("No local subnets in tenant {} for EP {}.", endpoint.getTenant(), endpoint.getKey()); + continue; } - NetworkDomainId epNetworkContainment= getEPNetworkContainment(endpoint); + NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint); for (Subnet subnet : subnets) { if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) { localSubnets.add(subnet); @@ -1085,24 +1482,6 @@ public class DestinationMapper extends FlowTable { return localSubnets; } - /** - * Reads data from datastore as synchronous call. - * - * @return {@link Optional#isPresent()} is {@code true} if reading was - * successful and data exists in datastore; - * {@link Optional#isPresent()} is {@code false} otherwise - */ - public static Optional readFromDs(LogicalDatastoreType store, InstanceIdentifier path, - ReadTransaction rTx) { - CheckedFuture, ReadFailedException> resultFuture = rTx.read(store, path); - try { - return resultFuture.checkedGet(); - } catch (ReadFailedException e) { - LOG.warn("Read failed from DS.", e); - return Optional.absent(); - } - } - static byte[] bytesFromHexString(String values) { String target = ""; if (values != null) {