X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=renderers%2Fofoverlay%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fgroupbasedpolicy%2Frenderer%2Fofoverlay%2Fflow%2FDestinationMapper.java;h=dc8e608c1a371ee0d863380c992e32ad531de8ab;hb=308d77bccb620c7cf580e0bb983610070bf20150;hp=b779fc5ef33c0162aa0302c4f77c9a7bc28947f2;hpb=d84bddb997cbc0c238588a006964362a555cb64b;p=groupbasedpolicy.git diff --git a/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java b/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java index b779fc5ef..dc8e608c1 100644 --- a/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java +++ b/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved. + * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved. * * This program and the accompanying materials are made available under the * terms of the Eclipse Public License v1.0 which accompanies this distribution, @@ -25,24 +25,30 @@ import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtil import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction; -import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveRegTunIdAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction; import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction; import java.math.BigInteger; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Set; import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction; +import org.opendaylight.controller.md.sal.binding.api.ReadTransaction; import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType; +import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; import org.opendaylight.groupbasedpolicy.endpoint.EpKey; import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext; import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.PolicyManager.FlowMap; @@ -50,6 +56,7 @@ import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMa import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals; import org.opendaylight.groupbasedpolicy.resolver.EgKey; import org.opendaylight.groupbasedpolicy.resolver.PolicyInfo; +import org.opendaylight.groupbasedpolicy.resolver.TenantUtils; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix; @@ -64,10 +71,18 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.M import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.EndpointLocation.LocationType; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.L3Context; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.Subnet; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId; @@ -78,130 +93,160 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder; -import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg0; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan; +import org.opendaylight.yangtools.yang.binding.DataObject; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Optional; import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; +import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; +import com.google.common.util.concurrent.CheckedFuture; /** * Manage the table that maps the destination address to the next hop for the * path as well as applies any relevant routing transformations. - * */ public class DestinationMapper extends FlowTable { - protected static final Logger LOG = - LoggerFactory.getLogger(DestinationMapper.class); - //TODO Li alagalah: Improve UT coverage for this class. + protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class); + + // TODO Li alagalah: Improve UT coverage for this class. - //TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows + // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows // This ensures we have the appropriate network-containment' public static final short TABLE_ID = 2; /** * This is the MAC address of the magical router in the sky */ - public static final MacAddress ROUTER_MAC = - new MacAddress("88:f0:31:b5:12:b5"); - public static final MacAddress MULTICAST_MAC = - new MacAddress("01:00:00:00:00:00"); + public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5"); + public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00"); public DestinationMapper(OfContext ctx) { super(ctx); } + Map> subnetsByTenant = new HashMap>(); + @Override public short getTableId() { return TABLE_ID; } @Override - public void sync(NodeId nodeId, PolicyInfo policyInfo, FlowMap flowMap) - throws Exception { - - flowMap.writeFlow(nodeId,TABLE_ID,dropFlow(Integer.valueOf(1), null)); - - Set visitedEps = new HashSet<>(); - Multimap subnetsByL3c = HashMultimap.create(); - Set fdIds = new HashSet<>(); - - for (EgKey epg : ctx.getEndpointManager().getGroupsForNode(nodeId)) { - Set peers = Sets.union(Collections.singleton(epg), - policyInfo.getPeers(epg)); - for (EgKey peer : peers) { - for(Endpoint epPeer : ctx.getEndpointManager().getEndpointsForGroup(peer)) { - syncEP(flowMap, nodeId, policyInfo, epPeer, visitedEps); - - //Process subnets and flood-domains for epPeer - EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, epPeer); - - subnetsByL3c.putAll(epOrds.getL3Id(),ctx.getPolicyResolver().getTenant(peer.getTenantId()).resolveSubnets(epOrds.getNetworkContainment())); - fdIds.add(epOrds.getFdId()); + public void sync(NodeId nodeId, PolicyInfo policyInfo, FlowMap flowMap) throws Exception { + + TenantId currentTenant; + + flowMap.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null)); + + SetMultimap visitedEps = HashMultimap.create(); + Set epOrdSet = new HashSet<>(); + + for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) { + Set srcEpgIds = new HashSet<>(); + if (srcEp.getEndpointGroup() != null) + srcEpgIds.add(srcEp.getEndpointGroup()); + if (srcEp.getEndpointGroups() != null) + srcEpgIds.addAll(srcEp.getEndpointGroups()); + + for (EndpointGroupId epgId : srcEpgIds) { + EgKey epg = new EgKey(srcEp.getTenant(), epgId); + Set peers = Sets.union(Collections.singleton(epg), policyInfo.getPeers(epg)); + for (EgKey peer : peers) { + for (Endpoint peerEp : ctx.getEndpointManager().getEndpointsForGroup(peer)) { + currentTenant = peerEp.getTenant(); + subnetsByTenant.put(currentTenant, getSubnets(currentTenant)); + EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress()); + EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress()); + + if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) { + continue; + } + syncEP(flowMap, nodeId, policyInfo, srcEp, peerEp); + visitedEps.put(srcEpKey, peerEpKey); + + // Process subnets and flood-domains for epPeer + EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, + peerEp); + epOrdSet.add(epOrds); + } } } } - // Write subnet flows - for (Integer subnetKey : subnetsByL3c.keySet()) { - for(Subnet sn:subnetsByL3c.get(subnetKey)) { - flowMap.writeFlow(nodeId, TABLE_ID, createRouterArpFlow(nodeId, sn, subnetKey)); + + for (Entry> subnetEntry : subnetsByTenant.entrySet()) { + if (subnetEntry.getValue() == null) { + LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey()); + continue; + } + currentTenant = subnetEntry.getKey(); + for (Subnet sn : subnetEntry.getValue()) { + L3Context l3c = getL3ContextForSubnet(currentTenant, sn); + Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn, + OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId())); + if (arpFlow != null) { + flowMap.writeFlow(nodeId, TABLE_ID, arpFlow); + } else { + LOG.debug( + "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .", + sn.getIpPrefix().getValue()); + } } } + // Write broadcast flows per flood domain. - for (Integer fdId : fdIds) { - if (groupExists(nodeId, fdId)) { - flowMap.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(fdId)); + for (EndpointFwdCtxOrdinals epOrd : epOrdSet) { + if (groupExists(nodeId, epOrd.getFdId())) { + flowMap.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd)); } } } + // set up next-hop destinations for all the endpoints in the endpoint // group on the node - - private Flow createBroadcastFlow(int fdId) { - FlowId flowId = new FlowId(new StringBuilder() - .append("broadcast|") - .append(fdId).toString()); + private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) { + FlowId flowId = new FlowId(new StringBuilder(). + append("broadcast|"). + append(epOrd.getFdId()). + toString()); MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(new EthernetMatchBuilder() - .setEthernetDestination(new EthernetDestinationBuilder() - .setAddress(MULTICAST_MAC) - .setMask(MULTICAST_MAC) - .build()) - .build()); - addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(fdId))); + .setEthernetMatch(new EthernetMatchBuilder() + .setEthernetDestination(new EthernetDestinationBuilder(). + setAddress(MULTICAST_MAC) + .setMask(MULTICAST_MAC).build()) + .build()); + addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId()))); + + FlowBuilder flowb = base().setPriority(Integer.valueOf(140)) + .setId(flowId) + .setMatch(mb.build()) + .setInstructions( + instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false), + groupAction(Long.valueOf(epOrd.getFdId()))))); - FlowBuilder flowb = base() - .setPriority(Integer.valueOf(140)) - .setId(flowId) - .setMatch(mb.build()) - .setInstructions(instructions(applyActionIns(nxMoveRegTunIdAction(NxmNxReg0.class, false), - groupAction(Long.valueOf(fdId))))); return flowb.build(); } - - private boolean groupExists(NodeId nodeId, Integer fdId) throws Exception { // Fetch existing GroupTables - if(ctx.getDataBroker()==null) { + if (ctx.getDataBroker() == null) { return false; } ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction(); InstanceIdentifier niid = createNodePath(nodeId); - Optional r = - t.read(LogicalDatastoreType.CONFIGURATION, niid).get(); + Optional r = t.read(LogicalDatastoreType.CONFIGURATION, niid).get(); if (!r.isPresent()) return false; FlowCapableNode fcn = r.get().getAugmentation(FlowCapableNode.class); @@ -219,71 +264,111 @@ public class DestinationMapper extends FlowTable { return false; } + private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) { + + if (ctx.getDataBroker() == null) { + return null; + } + + MacAddress defaultMacAddress = ROUTER_MAC; - private Flow createRouterArpFlow(NodeId nodeId, - Subnet sn, - int l3Id) { - if (sn != null && sn.getVirtualRouterIp() != null) { - if (sn.getVirtualRouterIp().getIpv4Address() != null) { - String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue(); - FlowId flowId = new FlowId(new StringBuffer() - .append("routerarp|") - .append(sn.getId().getValue()) - .append("|") - .append(ikey) - .append("|") - .append(l3Id) - .toString()); - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(ethernetMatch(null, null, ARP)) - .setLayer3Match(new ArpMatchBuilder() - .setArpOp(Integer.valueOf(1)) - .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32")) - .build()); - addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, - Long.valueOf(l3Id))); - BigInteger routerMac = - new BigInteger(1, bytesFromHexString(ROUTER_MAC - .getValue())); - FlowBuilder flowb = base() - .setPriority(150) - .setId(flowId) - .setMatch(mb.build()) - .setInstructions(instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), - setDlSrcAction(ROUTER_MAC), - nxLoadArpOpAction(BigInteger.valueOf(2L)), - nxMoveArpShaToArpThaAction(), - nxLoadArpShaAction(routerMac), - nxMoveArpSpaToArpTpaAction(), - nxLoadArpSpaAction(ikey), - outputAction(new NodeConnectorId(nodeId.getValue() + ":INPORT"))))); - return flowb.build(); + EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId()); + InstanceIdentifier endpointsIid = InstanceIdentifier.builder(Endpoints.class) + .child(EndpointL3.class, l3Key) + .build(); + ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction(); + + Optional r; + try { + r = t.read(LogicalDatastoreType.OPERATIONAL, endpointsIid).get(); + if (!r.isPresent()) + return defaultMacAddress; + EndpointL3 epL3 = r.get(); + if (epL3.getMacAddress() == null) { + return defaultMacAddress; } else { - LOG.warn("IPv6 virtual router {} for subnet {} not supported", - sn.getVirtualRouterIp(), sn.getId().getValue()); + return epL3.getMacAddress(); } + } catch (Exception e) { + LOG.error("Error reading EndpointL3 {}.{}", l3c, ipAddress, e); + return null; } - return null; } + private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) { + L3Context l3c = ctx.getPolicyResolver().getTenant(tenantId).resolveL3Context(sn.getId()); + return l3c; + } + + private Flow createRouterArpFlow(TenantId tenantId, NodeId nodeId, Subnet sn, int l3Id) { + if (sn == null || sn.getVirtualRouterIp() == null) { + LOG.trace("Didn't create routerArpFlow since either subnet or subnet virtual router was null"); + return null; + } + /* + * TODO: Li alagalah: This should be new Yang "gateways" list as well, + * that expresses the gateway and prefixes it is interface for. Should + * also check for external. + */ + if (sn.getVirtualRouterIp().getIpv4Address() != null) { + String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue(); + + L3Context l3c = getL3ContextForSubnet(tenantId, sn); + if (l3c == null) { + LOG.error("No L3 Context found associated with subnet {}", sn.getId()); + } + + MacAddress routerMac = routerPortMac(l3c, sn.getVirtualRouterIp()); + if (routerMac == null) { + return null; + } + + BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue())); + + FlowId flowId = new FlowId(new StringBuffer().append("routerarp|") + .append(sn.getId().getValue()) + .append("|") + .append(ikey) + .append("|") + .append(l3Id) + .toString()); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match( + new ArpMatchBuilder().setArpOp(Integer.valueOf(1)) + .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32")) + .build()); + addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id))); + + FlowBuilder flowb = base().setPriority(150) + .setId(flowId) + .setMatch(mb.build()) + .setInstructions( + instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac), + nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(), + nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(), + nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue() + + ":INPORT"))))); + return flowb.build(); + } else { + LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId() + .getValue()); + return null; + } + + } private Flow createLocalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) { - //TODO Li alagalah - refactor common code but keep simple method + // TODO Li alagalah - refactor common code but keep simple method ArrayList instructions = new ArrayList<>(); List applyActions = new ArrayList<>(); int order = 0; - Action setdEPG = nxLoadRegAction(NxmNxReg2.class, - BigInteger.valueOf(epFwdCtxOrds.getEpgId())); - Action setdCG = nxLoadRegAction(NxmNxReg3.class, - BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId())); Action setNextHop; String nextHop; - - // BEGIN L2 LOCAL nextHop = ofc.getNodeConnectorId().getValue(); @@ -291,80 +376,61 @@ public class DestinationMapper extends FlowTable { try { portNum = getOfPortNum(ofc.getNodeConnectorId()); } catch (NumberFormatException ex) { - LOG.warn("Could not parse port number {}", - ofc.getNodeConnectorId(), ex); + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); return null; } - setNextHop = nxLoadRegAction(NxmNxReg7.class, - BigInteger.valueOf(portNum)); + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); - //END L2 LOCAL + // END L2 LOCAL order += 1; applyActions.add(setdEPG); applyActions.add(setdCG); applyActions.add(setNextHop); - Instruction applyActionsIns = new InstructionBuilder() - .setOrder(order++) - .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) - .build(); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); instructions.add(applyActionsIns); - Instruction gotoTable = new InstructionBuilder() - .setOrder(order++) - .setInstruction(gotoTableIns((short) (getTableId() + 1))) - .build(); + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns((short) (getTableId() + 1))) + .build(); instructions.add(gotoTable); - - FlowId flowid = new FlowId(new StringBuilder() - .append(epFwdCtxOrds.getBdId()) - .append("|l2|") - .append(ep.getMacAddress().getValue()) - .append("|") - .append(nextHop) - .toString()); - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(ethernetMatch(null, - ep.getMacAddress(), - null)); + FlowId flowid = new FlowId(new StringBuilder().append(epFwdCtxOrds.getBdId()) + .append("|l2|") + .append(ep.getMacAddress().getValue()) + .append("|") + .append(nextHop) + .toString()); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null)); addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId()))); - FlowBuilder flowb = base() - .setId(flowid) - .setPriority(Integer.valueOf(50)) - .setMatch(mb.build()) - .setInstructions(new InstructionsBuilder() - .setInstruction(instructions) - .build()); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(50)) + .setMatch(mb.build()) + .setInstructions(new InstructionsBuilder().setInstruction(instructions).build()); return flowb.build(); } - - private void syncEP(FlowMap flowMap, - NodeId nodeId, PolicyInfo policyInfo, - Endpoint epPeer, Set visitedEps) + private void syncEP(FlowMap flowMap, NodeId nodeId, PolicyInfo policyInfo, Endpoint srcEp, Endpoint destEp) throws Exception { - EpKey epPeerKey = new EpKey(epPeer.getL2Context(),epPeer.getMacAddress()); + // TODO: Conditions messed up, but for now, send policyInfo until this + // is fixed. + EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, destEp); + EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, srcEp); - if (visitedEps.contains(epPeerKey)) { + if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) { + LOG.trace("Didn't process endpoint due to either tenant, or EPG(s) being null", destEp.getKey()); return; } - visitedEps.add(epPeerKey); - - // TODO: Conditions messed up, but for now, send policyInfo until this is fixed. - EndpointFwdCtxOrdinals epFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, epPeer); + OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class); - if (epPeer.getTenant() == null || (epPeer.getEndpointGroup() == null && epPeer.getEndpointGroups() == null)) - return; - OfOverlayContext ofc = epPeer.getAugmentation(OfOverlayContext.class); - - - - ////////////////////////////////////////////////////////////////////////////////////////// + // //////////////////////////////////////////////////////////////////////////////////////// /* - * NOT HANDLING EXTERNALS + * NOT HANDLING EXTERNALS TODO: alagalah Li: External Gateway + * functionality needed here. */ if (LocationType.External.equals(ofc.getLocationType())) { // XXX - TODO - perform NAT and send to the external network @@ -373,59 +439,155 @@ public class DestinationMapper extends FlowTable { return; } + /* + * Only care about subnets for L3, but fetch them before loop. We need + * the local subnets for setting SRC MAC for routing. All Routing is now + * done locally! YAY! Instead of being shovelled L2 style across network + * ala Helium. + */ + List localSubnets = getLocalSubnets(nodeId); + if (localSubnets == null) { + LOG.error("No subnets could be found locally for node: {}", nodeId); + return; + } + if (Objects.equals(ofc.getNodeId(), nodeId)) { // this is a local endpoint; send to the approppriate local // port - flowMap.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(epPeer,epFwdCtxOrds,ofc)); - - if (epPeer.getL3Address() == null) + if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) { + flowMap.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc)); + } + // TODO Li alagalah: Need to move to EndpointL3 for L3 processing. + // The Endpoint conflation must end! + if (destEp.getL3Address() == null) { + LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey()); return; - for (L3Address l3a : epPeer.getL3Address()) { + } + + for (L3Address l3a : destEp.getL3Address()) { if (l3a.getIpAddress() == null || l3a.getL3Context() == null) { - LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",epPeer.getL3Address().toString()); + LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}", + destEp.getL3Address()); continue; } else { - flowMap.writeFlow(nodeId, TABLE_ID, createLocalL3Flow(epPeer,l3a, epFwdCtxOrds,ofc)); + for (Subnet localSubnet : localSubnets) { + Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet); + if (flow != null) { + flowMap.writeFlow(nodeId, TABLE_ID, flow); + } else { + LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(), + localSubnet.getIpPrefix().getValue()); + } + } } } } else { // this endpoint is on a different switch; send to the // appropriate tunnel - - Flow remoteL2Flow = createRemoteL2Flow(epPeer, nodeId, epFwdCtxOrds, ofc); - if (remoteL2Flow != null) { - flowMap.writeFlow(nodeId, TABLE_ID, remoteL2Flow); + if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) { + Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc); + if (remoteL2Flow != null) { + flowMap.writeFlow(nodeId, TABLE_ID, remoteL2Flow); + } + } else { + LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}", + srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId()); } - if (epPeer.getL3Address() == null) + // TODO Li alagalah: Need to move to EndpointL3 for L3 processing. + // The Endpoint conflation must end! + if (destEp.getL3Address() == null) { + LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey()); return; - for (L3Address l3a : epPeer.getL3Address()) { + } + for (L3Address l3a : destEp.getL3Address()) { if (l3a.getIpAddress() == null || l3a.getL3Context() == null) { - LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",epPeer.getL3Address().toString()); + LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}", + destEp.getL3Address()); continue; } else { - Flow remoteL3Flow = createRemoteL3Flow(l3a, nodeId,epFwdCtxOrds, ofc); - if (remoteL3Flow != null) { - flowMap.writeFlow(nodeId, TABLE_ID, remoteL3Flow); + for (Subnet localSubnet : localSubnets) { + Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds, + destEpFwdCtxOrds, ofc, localSubnet); + if (remoteL3Flow != null) { + flowMap.writeFlow(nodeId, TABLE_ID, remoteL3Flow); + } else { + LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(), + localSubnet.getIpPrefix().getValue()); + } } } } } // remote (tunnel) - - // } } - /* ################################## - * DestMapper Flow methods + /* + * ################################## DestMapper Flow methods * ################################## */ - private Flow createLocalL3Flow(Endpoint ep,L3Address l3a, EndpointFwdCtxOrdinals epFwdCtxOrds,OfOverlayContext ofc) { + private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds, + OfOverlayContext ofc, Subnet srcSubnet) { - //TODO Li alagalah - refactor common code but keep simple method + // TODO Li alagalah - refactor common code but keep simple method + + Subnet destSubnet = null; + HashSet subnets = getSubnets(destEp.getTenant()); + if (subnets == null) { + LOG.trace("No subnets in tenant {}", destL3Address.getIpAddress()); + return null; + } + NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp); + for (Subnet subnet : subnets) { + // TODO Li alagalah add IPv6 support + if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) { + destSubnet = subnet; + break; + } + } + if (destSubnet == null) { + LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress()); + return null; + } + + if (destSubnet.getVirtualRouterIp() == null) { + LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(), + destL3Address.getKey()); + return null; + } + + if (srcSubnet.getVirtualRouterIp() == null) { + LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix()); + return null; + } + L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet); + if (destL3c == null || destL3c.getId() == null) { + LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId()); + return null; + } + L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet); + if (srcL3c == null || srcL3c.getId() == null) { + LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId()); + return null; + } + + if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) { + LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId() + .getValue(), destL3c.getId().getValue()); + return null; + } + + MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp()); + MacAddress epDestMac = destEp.getMacAddress(); + MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp()); + + if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) { + // This is our final destination, so match on actual EP mac. + matcherMac = epDestMac; + } ArrayList l3instructions = new ArrayList<>(); List applyActions = new ArrayList<>(); @@ -433,16 +595,11 @@ public class DestinationMapper extends FlowTable { int order = 0; - Action setdEPG = nxLoadRegAction(NxmNxReg2.class, - BigInteger.valueOf(epFwdCtxOrds.getEpgId())); - Action setdCG = nxLoadRegAction(NxmNxReg3.class, - BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId())); Action setNextHop; String nextHop; - Action setDlSrc = setDlSrcAction(ROUTER_MAC); - Action decTtl = decNwTtlAction(); - // BEGIN L3 LOCAL nextHop = ofc.getNodeConnectorId().getValue(); @@ -450,88 +607,84 @@ public class DestinationMapper extends FlowTable { try { portNum = getOfPortNum(ofc.getNodeConnectorId()); } catch (NumberFormatException ex) { - LOG.warn("Could not parse port number {}", - ofc.getNodeConnectorId(), ex); + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); return null; } - setNextHop = nxLoadRegAction(NxmNxReg7.class, - BigInteger.valueOf(portNum)); + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); + // END L3 LOCAL - Action setDlDst = setDlDstAction(ep.getMacAddress()); + // Lets not re-write the srcMac if its local. + if (!(matcherMac.getValue().equals(epDestMac.getValue()))) { + Action setDlSrc = setDlSrcAction(destSubnetGatewayMac); + l3ApplyActions.add(setDlSrc); + } + + Action setDlDst = setDlDstAction(epDestMac); l3ApplyActions.add(setDlDst); - //END L3 LOCAL - l3ApplyActions.add(setDlSrc); + Action decTtl = decNwTtlAction(); l3ApplyActions.add(decTtl); + order += 1; applyActions.add(setdEPG); applyActions.add(setdCG); applyActions.add(setNextHop); applyActions.addAll(l3ApplyActions); - Instruction applyActionsIns = new InstructionBuilder() - .setOrder(order++) - .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) - .build(); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); l3instructions.add(applyActionsIns); - Instruction gotoTable = new InstructionBuilder() - .setOrder(order++) - .setInstruction(gotoTableIns((short) (getTableId() + 1))) - .build(); + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns((short) (getTableId() + 1))) + .build(); l3instructions.add(gotoTable); Layer3Match m = null; Long etherType = null; String ikey = null; - if (l3a.getIpAddress().getIpv4Address() != null) { - ikey = l3a.getIpAddress().getIpv4Address().getValue() + "/32"; + if (destL3Address.getIpAddress().getIpv4Address() != null) { + ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32"; etherType = IPv4; - m = new Ipv4MatchBuilder() - .setIpv4Destination(new Ipv4Prefix(ikey)) - .build(); - } else if (l3a.getIpAddress().getIpv6Address() != null) { - ikey = l3a.getIpAddress().getIpv6Address().getValue() + "/128"; + m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build(); + } else if (destL3Address.getIpAddress().getIpv6Address() != null) { + ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128"; etherType = IPv6; - m = new Ipv6MatchBuilder() - .setIpv6Destination(new Ipv6Prefix(ikey)) - .build(); + m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build(); } else { - LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.",l3a.toString()); + LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString()); return null; } - FlowId flowid = new FlowId(new StringBuilder() - .append(Integer.toString(epFwdCtxOrds.getL3Id())) - .append("|l3|") - .append(ikey) - .append("|") - .append(Integer.toString(epFwdCtxOrds.getEpgId())) - .append(" ") - .append(Integer.toString(epFwdCtxOrds.getCgId())) - .append("|") - .append(nextHop) - .toString()); - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(ethernetMatch(null, - ROUTER_MAC, - etherType)) - .setLayer3Match(m); - addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, - Long.valueOf(epFwdCtxOrds.getL3Id()))); - FlowBuilder flowb = base() - .setId(flowid) - .setPriority(Integer.valueOf(132)) - .setMatch(mb.build()) - .setInstructions(new InstructionsBuilder() - .setInstruction(l3instructions) - .build()); + FlowId flowid = new FlowId(new StringBuilder().append(Integer.toString(epFwdCtxOrds.getL3Id())) + .append("|l3|") + .append(ikey) + .append("|") + .append(Integer.toString(epFwdCtxOrds.getEpgId())) + .append("|") + .append(Integer.toString(epFwdCtxOrds.getCgId())) + .append("|") + .append(matcherMac) + .append("|") + .append(destSubnetGatewayMac) + .append("|") + .append(nextHop) + .toString()); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType)) + .setLayer3Match(m); + addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id()))); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(132)) + .setMatch(mb.build()) + .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build()); return flowb.build(); } - private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) { + private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds, + EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) { - //TODO Li alagalah - refactor common code but keep simple method + // TODO Li alagalah - refactor common code but keep simple method // this endpoint is on a different switch; send to the // appropriate tunnel @@ -539,21 +692,16 @@ public class DestinationMapper extends FlowTable { ArrayList instructions = new ArrayList<>(); List applyActions = new ArrayList<>(); - int order = 0; - Action setdEPG = nxLoadRegAction(NxmNxReg2.class, - BigInteger.valueOf(epFwdCtxOrds.getEpgId())); - Action setdCG = nxLoadRegAction(NxmNxReg3.class, - BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId())); Action setNextHop; String nextHop; // BEGIN TUNNEL HANDLING - IpAddress tunDst = - ctx.getSwitchManager().getTunnelIP(ofc.getNodeId()); - NodeConnectorId tunPort = - ctx.getSwitchManager().getTunnelPort(nodeId); + IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class); + NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class); if (tunDst == null) { LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep); return null; @@ -570,8 +718,7 @@ public class DestinationMapper extends FlowTable { tundstAction = nxLoadTunIPv4Action(nextHop, false); } else if (tunDst.getIpv6Address() != null) { // nextHop = tunDst.getIpv6Address().getValue(); - LOG.error("IPv6 tunnel destination {} for {} not supported", - tunDst.getIpv6Address().getValue(), + LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(), ofc.getNodeId()); return null; } else { @@ -584,17 +731,11 @@ public class DestinationMapper extends FlowTable { try { portNum = getOfPortNum(tunPort); } catch (NumberFormatException ex) { - LOG.warn("Could not parse port number {}", - ofc.getNodeConnectorId(), ex); + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); return null; } - setNextHop = nxLoadRegAction(NxmNxReg7.class, - BigInteger.valueOf(portNum)); - Action tunIdAction = - nxMoveRegTunIdAction(NxmNxReg0.class, false); - - applyActions.add(tunIdAction); + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); applyActions.add(tundstAction); // END TUNNEL @@ -602,83 +743,116 @@ public class DestinationMapper extends FlowTable { applyActions.add(setdEPG); applyActions.add(setdCG); applyActions.add(setNextHop); - Instruction applyActionsIns = new InstructionBuilder() - .setOrder(order++) - .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) - .build(); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); instructions.add(applyActionsIns); - applyActionsIns = new InstructionBuilder() - .setOrder(order++) - .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) - .build(); + applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); - Instruction gotoTable = new InstructionBuilder() - .setOrder(order++) - .setInstruction(gotoTableIns((short) (getTableId() + 1))) - .build(); + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns((short) (getTableId() + 1))) + .build(); instructions.add(gotoTable); - FlowId flowid = new FlowId(new StringBuilder() - .append(epFwdCtxOrds.getBdId()) - .append("|l2|") - .append(ep.getMacAddress().getValue()) - .append("|") - .append(nextHop) - .toString()); - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(ethernetMatch(null, - ep.getMacAddress(), - null)); - addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId()))); - FlowBuilder flowb = base() - .setId(flowid) - .setPriority(Integer.valueOf(50)) - .setMatch(mb.build()) - .setInstructions(new InstructionsBuilder() - .setInstruction(instructions) - .build()); + FlowId flowid = new FlowId(new StringBuilder().append(destEpFwdCtxOrds.getBdId()) + .append("|l2|") + .append(ep.getMacAddress().getValue()) + .append("|") + .append(srcEpFwdCtxOrds.getBdId()) + .append("|") + .append(nextHop) + .toString()); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null)); + addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId()))); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(50)) + .setMatch(mb.build()) + .setInstructions(new InstructionsBuilder().setInstruction(instructions).build()); return flowb.build(); } + private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId, + EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc, + Subnet srcSubnet) { - private Flow createRemoteL3Flow(L3Address l3a, NodeId nodeId, EndpointFwdCtxOrdinals epFwdCtxOrds,OfOverlayContext ofc) { - - //TODO Li alagalah - refactor common code but keep simple method - + // TODO Li alagalah - refactor common code but keep simple method // this endpoint is on a different switch; send to the // appropriate tunnel + Subnet destSubnet = null; + HashSet subnets = getSubnets(destEp.getTenant()); + if (subnets == null) { + LOG.trace("No subnets in tenant {}", destL3Address.getIpAddress()); + return null; + } + NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp); + for (Subnet subnet : subnets) { + // TODO Li alagalah add IPv6 support + if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) { + destSubnet = subnet; + break; + } + } + if (destSubnet == null) { + LOG.info("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress()); + return null; + } + + if (destSubnet.getVirtualRouterIp() == null) { + LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(), + destL3Address.getKey()); + return null; + } + + if (srcSubnet.getVirtualRouterIp() == null) { + LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix()); + return null; + } + L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet); + if (destL3c == null || destL3c.getId() == null) { + LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId()); + return null; + } + L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet); + if (srcL3c == null || srcL3c.getId() == null) { + LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId()); + return null; + } + + if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) { + LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId() + .getValue(), destL3c.getId().getValue()); + return null; + } + + MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp()); + MacAddress epDestMac = destEp.getMacAddress(); + MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp()); ArrayList l3instructions = new ArrayList<>(); List applyActions = new ArrayList<>(); List l3ApplyActions = new ArrayList<>(); - int order = 0; - Action setdEPG = nxLoadRegAction(NxmNxReg2.class, - BigInteger.valueOf(epFwdCtxOrds.getEpgId())); - Action setdCG = nxLoadRegAction(NxmNxReg3.class, - BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId())); Action setNextHop; String nextHop; - Action setDlSrc = setDlSrcAction(ROUTER_MAC); - Action decTtl = decNwTtlAction(); - // BEGIN TUNNEL HANDLING - IpAddress tunDst = - ctx.getSwitchManager().getTunnelIP(ofc.getNodeId()); - NodeConnectorId tunPort = - ctx.getSwitchManager().getTunnelPort(nodeId); + IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class); + NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class); if (tunDst == null) { - LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, l3a); + LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address); return null; } if (tunPort == null) { - LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, l3a); + LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, destL3Address); return null; } @@ -689,8 +863,7 @@ public class DestinationMapper extends FlowTable { tundstAction = nxLoadTunIPv4Action(nextHop, false); } else if (tunDst.getIpv6Address() != null) { // nextHop = tunDst.getIpv6Address().getValue(); - LOG.error("IPv6 tunnel destination {} for {} not supported", - tunDst.getIpv6Address().getValue(), + LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(), ofc.getNodeId()); return null; } else { @@ -703,90 +876,165 @@ public class DestinationMapper extends FlowTable { try { portNum = getOfPortNum(tunPort); } catch (NumberFormatException ex) { - LOG.warn("Could not parse port number {}", - ofc.getNodeConnectorId(), ex); + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); return null; } - setNextHop = nxLoadRegAction(NxmNxReg7.class, - BigInteger.valueOf(portNum)); - Action tunIdAction = - nxMoveRegTunIdAction(NxmNxReg0.class, false); - - applyActions.add(tunIdAction); + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); applyActions.add(tundstAction); // END TUNNEL - - l3ApplyActions.add(setDlSrc); - l3ApplyActions.add(decTtl); order += 1; applyActions.add(setdEPG); applyActions.add(setdCG); applyActions.add(setNextHop); + + Action setDlSrc = setDlSrcAction(destSubnetGatewayMac); l3ApplyActions.add(setDlSrc); + + Action setDlDst = setDlDstAction(epDestMac); + l3ApplyActions.add(setDlDst); + + Action decTtl = decNwTtlAction(); l3ApplyActions.add(decTtl); applyActions.addAll(l3ApplyActions); - Instruction applyActionsIns = new InstructionBuilder() - .setOrder(order++) - .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) - .build(); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); l3instructions.add(applyActionsIns); - Instruction gotoTable = new InstructionBuilder() - .setOrder(order++) - .setInstruction(gotoTableIns((short) (getTableId() + 1))) - .build(); + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns((short) (getTableId() + 1))) + .build(); l3instructions.add(gotoTable); Layer3Match m = null; Long etherType = null; String ikey = null; - if (l3a.getIpAddress().getIpv4Address() != null) { - ikey = l3a.getIpAddress().getIpv4Address().getValue() + "/32"; + if (destL3Address.getIpAddress().getIpv4Address() != null) { + ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32"; etherType = IPv4; - m = new Ipv4MatchBuilder() - .setIpv4Destination(new Ipv4Prefix(ikey)) - .build(); - } else if (l3a.getIpAddress().getIpv6Address() != null) { - ikey = l3a.getIpAddress().getIpv6Address().getValue() + "/128"; + m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build(); + } else if (destL3Address.getIpAddress().getIpv6Address() != null) { + ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128"; etherType = IPv6; - m = new Ipv6MatchBuilder() - .setIpv6Destination(new Ipv6Prefix(ikey)) - .build(); + m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build(); } else { - LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.",l3a.toString()); + LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString()); return null; } - FlowId flowid = new FlowId(new StringBuilder() - .append(Integer.toString(epFwdCtxOrds.getL3Id())) - .append("|l3|") - .append(ikey) - .append("|") - .append(Integer.toString(epFwdCtxOrds.getEpgId())) - .append(" ") - .append(Integer.toString(epFwdCtxOrds.getCgId())) - .append("|") - .append(nextHop) - .toString()); - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(ethernetMatch(null, - ROUTER_MAC, - etherType)) - .setLayer3Match(m); - addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, - Long.valueOf(epFwdCtxOrds.getL3Id()))); - FlowBuilder flowb = base() - .setId(flowid) - .setPriority(Integer.valueOf(132)) - .setMatch(mb.build()) - .setInstructions(new InstructionsBuilder() - .setInstruction(l3instructions) - .build()); + FlowId flowid = new FlowId(new StringBuilder().append(Integer.toString(destEpFwdCtxOrds.getL3Id())) + .append("|l3|") + .append(ikey) + .append("|") + .append(matcherMac) + .append("|") + .append(destSubnetGatewayMac) + .append("|") + .append(srcEpFwdCtxOrds.getL3Id()) + .append("|") + .append(nextHop) + .toString()); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType)) + .setLayer3Match(m); + addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id()))); + + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(132)) + .setMatch(mb.build()) + .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build()); return flowb.build(); } + private NetworkDomainId getEPNetworkContainment(Endpoint endpoint) { + if (endpoint.getNetworkContainment() != null) { + return endpoint.getNetworkContainment(); + } else { + /* + * TODO: Be alagalah: Endpoint Refactor: This should be set on input + * which we can't do because of the backwards way endpoints were + * "architected". + */ + return ctx.getPolicyResolver() + .getTenant(endpoint.getTenant()) + .getEndpointGroup(endpoint.getEndpointGroup()) + .getNetworkDomain(); + } + } + + private HashSet getSubnets(final TenantId tenantId) { + + // if (subnetsByTenant.get(tenantId) != null) { + // return subnetsByTenant.get(tenantId); + // } + + if (ctx.getDataBroker() == null) { + return null; + } + + ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction(); + InstanceIdentifier tiid = TenantUtils.tenantIid(tenantId); + Optional tenantInfo; + try { + tenantInfo = t.read(LogicalDatastoreType.CONFIGURATION, tiid).get(); + } catch (Exception e) { + LOG.error("Could not read Tenant {}", tenantId, e); + return null; + } + + HashSet subnets = new HashSet(); + + if (!tenantInfo.isPresent()) { + LOG.warn("Tenant {} not found", tenantId); + return null; + } + + subnets.addAll(tenantInfo.get().getSubnet()); + // subnetsByTenant.put(tenantId, subnets); + return subnets; + } + + // Need a method to get subnets for EPs attached to the node locally + // to set the source Mac address for the router interface. + private List getLocalSubnets(NodeId nodeId) { + Collection endpointsForNode = ctx.getEndpointManager().getEndpointsForNode(nodeId); + + List localSubnets = new ArrayList(); + + for (Endpoint endpoint : endpointsForNode) { + HashSet subnets = getSubnets(endpoint.getTenant()); + if (subnets == null) { + LOG.error("No local subnets."); + return null; + } + NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint); + for (Subnet subnet : subnets) { + if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) { + localSubnets.add(subnet); + } + } + } + return localSubnets; + } + + /** + * Reads data from datastore as synchronous call. + * + * @return {@link Optional#isPresent()} is {@code true} if reading was + * successful and data exists in datastore; {@link Optional#isPresent()} is + * {@code false} otherwise + */ + public static Optional readFromDs(LogicalDatastoreType store, InstanceIdentifier path, + ReadTransaction rTx) { + CheckedFuture, ReadFailedException> resultFuture = rTx.read(store, path); + try { + return resultFuture.checkedGet(); + } catch (ReadFailedException e) { + LOG.warn("Read failed from DS.", e); + return Optional.absent(); + } + } static byte[] bytesFromHexString(String values) { String target = "";