X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;ds=sidebyside;f=renderers%2Fofoverlay%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fgroupbasedpolicy%2Frenderer%2Fofoverlay%2Fflow%2FDestinationMapper.java;h=9f8e5a328ba07d17ad262ef76cf88420e14ed8e8;hb=5045a1327bba703fd30861ddd96f0926e0354544;hp=a519b98f1a49cc4b83d6403eab5f52040c172748;hpb=5c520e28ac4a5227b4b0d22cf9aff1294c19dbe9;p=groupbasedpolicy.git diff --git a/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java b/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java index a519b98f1..9f8e5a328 100644 --- a/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java +++ b/renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved. + * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved. * * This program and the accompanying materials are made available under the * terms of the Eclipse Public License v1.0 which accompanies this distribution, @@ -8,26 +8,55 @@ package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.createNodePath; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.gotoTableIns; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.groupAction; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.instructions; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpOpAction; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpShaAction; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction; +import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction; + import java.math.BigInteger; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Set; import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction; -import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction; +import org.opendaylight.controller.md.sal.binding.api.ReadTransaction; import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType; +import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; +import org.opendaylight.groupbasedpolicy.endpoint.EpKey; import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext; -import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.PolicyManager.Dirty; +import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.PolicyManager.FlowMap; import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch; -import org.opendaylight.groupbasedpolicy.resolver.ConditionGroup; +import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals; import org.opendaylight.groupbasedpolicy.resolver.EgKey; -import org.opendaylight.groupbasedpolicy.resolver.IndexedTenant; import org.opendaylight.groupbasedpolicy.resolver.PolicyInfo; +import org.opendaylight.groupbasedpolicy.resolver.TenantUtils; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix; @@ -35,21 +64,24 @@ import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types. import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId; -import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group; -import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.ConditionName; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.EndpointLocation.LocationType; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext; -import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.EndpointGroup; -import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.L2BridgeDomain; -import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.L2FloodDomain; +import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.L3Context; import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.Subnet; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId; @@ -61,431 +93,944 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder; -import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg0; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6; import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan; +import org.opendaylight.yangtools.yang.binding.DataObject; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Optional; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; - -import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.*; +import com.google.common.util.concurrent.CheckedFuture; /** - * Manage the table that maps the destination address to the next hop - * for the path as well as applies any relevant routing transformations. - * @author readams + * Manage the table that maps the destination address to the next hop for the + * path as well as applies any relevant routing transformations. */ public class DestinationMapper extends FlowTable { - protected static final Logger LOG = - LoggerFactory.getLogger(DestinationMapper.class); + + protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class); + + // TODO Li alagalah: Improve UT coverage for this class. + + // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows + // This ensures we have the appropriate network-containment' public static final short TABLE_ID = 2; /** * This is the MAC address of the magical router in the sky */ - public static final MacAddress ROUTER_MAC = - new MacAddress("88:f0:31:b5:12:b5"); - public static final MacAddress MULTICAST_MAC = - new MacAddress("01:00:00:00:00:00"); + public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5"); + public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00"); public DestinationMapper(OfContext ctx) { super(ctx); } + Map> subnetsByTenant = new HashMap>(); + @Override public short getTableId() { return TABLE_ID; } @Override - public void sync(ReadWriteTransaction t, - InstanceIdentifier tiid, - Map flowMap, - NodeId nodeId, PolicyInfo policyInfo, Dirty dirty) - throws Exception { - dropFlow(t, tiid, flowMap, Integer.valueOf(1), null); - - HashSet visitedEgs = new HashSet<>(); - HashSet visitedFds = new HashSet<>(); - - for (EgKey epg : ctx.getEndpointManager().getGroupsForNode(nodeId)) { - Set peers = Sets.union(Collections.singleton(epg), - policyInfo.getPeers(epg)); - for (EgKey peer : peers) { - syncEPG(t, tiid, flowMap, nodeId, - policyInfo, peer, - visitedEgs, visitedFds); + public void sync(NodeId nodeId, PolicyInfo policyInfo, FlowMap flowMap) throws Exception { + + TenantId currentTenant; + + flowMap.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null)); + + SetMultimap visitedEps = HashMultimap.create(); + Set epOrdSet = new HashSet<>(); + + for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) { + Set srcEpgIds = new HashSet<>(); + if (srcEp.getEndpointGroup() != null) + srcEpgIds.add(srcEp.getEndpointGroup()); + if (srcEp.getEndpointGroups() != null) + srcEpgIds.addAll(srcEp.getEndpointGroups()); + + for (EndpointGroupId epgId : srcEpgIds) { + EgKey epg = new EgKey(srcEp.getTenant(), epgId); + Set peers = Sets.union(Collections.singleton(epg), policyInfo.getPeers(epg)); + for (EgKey peer : peers) { + for (Endpoint peerEp : ctx.getEndpointManager().getEndpointsForGroup(peer)) { + currentTenant = peerEp.getTenant(); + subnetsByTenant.put(currentTenant, getSubnets(currentTenant)); + EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress()); + EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress()); + + if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) { + continue; + } + syncEP(flowMap, nodeId, policyInfo, srcEp, peerEp); + visitedEps.put(srcEpKey, peerEpKey); + + // Process subnets and flood-domains for epPeer + EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, + peerEp); + epOrdSet.add(epOrds); + } + } } } - } - // set up next-hop destinations for all the endpoints in the endpoint - // group on the node - private void syncEPG(ReadWriteTransaction t, - InstanceIdentifier
tiid, - Map flowMap, - NodeId nodeId, PolicyInfo policyInfo, - EgKey key, - HashSet visitedEgs, - HashSet visitedFds) throws Exception { - if (visitedEgs.contains(key)) return; - visitedEgs.add(key); - - IndexedTenant tenant = ctx.getPolicyResolver().getTenant(key.getTenantId()); - EndpointGroup eg = tenant.getEndpointGroup(key.getEgId()); - L2FloodDomain fd = tenant.resolveL2FloodDomain(eg.getNetworkDomain()); - Collection sns = tenant.resolveSubnets(eg.getNetworkDomain()); - L3Context l3c = tenant.resolveL3Context(eg.getNetworkDomain()); - int l3Id = 0; - - if (l3c != null) - l3Id = ctx.getPolicyManager().getContextOrdinal(key.getTenantId(), - l3c.getId()); - - Collection egEps = ctx.getEndpointManager() - .getEndpointsForGroup(key); - - for (Endpoint e : egEps) { - if (e.getTenant() == null || e.getEndpointGroup() == null) + for (Entry> subnetEntry : subnetsByTenant.entrySet()) { + if (subnetEntry.getValue() == null) { + LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey()); continue; - OfOverlayContext ofc = e.getAugmentation(OfOverlayContext.class); - if (ofc == null || ofc.getNodeId() == null) continue; - - syncEP(t, tiid, flowMap, nodeId, policyInfo, e, ofc, tenant, key); - } - - if (fd == null) return; - Integer fdId = ctx.getPolicyManager().getContextOrdinal(key.getTenantId(), - fd.getId()); - if (visitedFds.contains(fdId)) return; - visitedFds.add(fdId); - - //GroupTable must exist before we start adding flows that direct to it via fdId - if(groupExists(nodeId,fdId)) { - - FlowId flowId = new FlowId(new StringBuilder() - .append("broadcast|") - .append(fdId).toString()); - if (visit(flowMap, flowId.getValue())) { - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(new EthernetMatchBuilder() - .setEthernetDestination(new EthernetDestinationBuilder() - .setAddress(MULTICAST_MAC) - .setMask(MULTICAST_MAC) - .build()) - .build()); - addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class,Long.valueOf(fdId))); - - FlowBuilder flow = base() - .setPriority(Integer.valueOf(140)) - .setId(flowId) - .setMatch(mb.build()) - .setInstructions(instructions(applyActionIns(nxMoveRegTunIdAction(NxmNxReg0.class, false), - groupAction(Long.valueOf(fdId))))); - writeFlow(t, tiid, flow.build()); + } + currentTenant = subnetEntry.getKey(); + for (Subnet sn : subnetEntry.getValue()) { + L3Context l3c = getL3ContextForSubnet(currentTenant, sn); + Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn, + OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId())); + if (arpFlow != null) { + flowMap.writeFlow(nodeId, TABLE_ID, arpFlow); + } else { + LOG.debug( + "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .", + sn.getIpPrefix().getValue()); + } } } - for (Subnet sn : sns) { - writeRouterArpFlow(t, tiid, flowMap, nodeId, sn, l3Id); + + // Write broadcast flows per flood domain. + for (EndpointFwdCtxOrdinals epOrd : epOrdSet) { + if (groupExists(nodeId, epOrd.getFdId())) { + flowMap.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd)); + } } } + // set up next-hop destinations for all the endpoints in the endpoint + // group on the node + + private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) { + FlowId flowId = new FlowId("broadcast|" + epOrd.getFdId()); + MatchBuilder mb = new MatchBuilder() + .setEthernetMatch(new EthernetMatchBuilder() + .setEthernetDestination(new EthernetDestinationBuilder(). + setAddress(MULTICAST_MAC) + .setMask(MULTICAST_MAC).build()) + .build()); + addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId()))); + + FlowBuilder flowb = base().setPriority(Integer.valueOf(140)) + .setId(flowId) + .setMatch(mb.build()) + .setInstructions( + instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false), + groupAction(Long.valueOf(epOrd.getFdId()))))); + + return flowb.build(); + } + private boolean groupExists(NodeId nodeId, Integer fdId) throws Exception { - //Fetch existing GroupTables - if(ctx.getDataBroker()==null) return false; + // Fetch existing GroupTables + if (ctx.getDataBroker() == null) { + return false; + } ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction(); InstanceIdentifier niid = createNodePath(nodeId); - Optional r = - t.read(LogicalDatastoreType.CONFIGURATION, niid).get(); - if (!r.isPresent()) return false; + Optional r = t.read(LogicalDatastoreType.CONFIGURATION, niid).get(); + if (!r.isPresent()) + return false; FlowCapableNode fcn = r.get().getAugmentation(FlowCapableNode.class); - if (fcn == null) return false; + if (fcn == null) + return false; if (fcn.getGroup() != null) { for (Group g : fcn.getGroup()) { - if (g.getGroupId().getValue().equals(Long.valueOf(fdId))) { //Group Exists. + if (g.getGroupId().getValue().equals(Long.valueOf(fdId))) { // Group + // Exists. return true; } - } } return false; } + private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) { - private void writeRouterArpFlow(ReadWriteTransaction t, - InstanceIdentifier
tiid, - Map flowMap, - NodeId nodeId, - Subnet sn, - int l3Id) { - if (sn != null && sn.getVirtualRouterIp() != null) { - if (sn.getVirtualRouterIp().getIpv4Address() != null) { - String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue(); - FlowId flowId = new FlowId(new StringBuffer() - .append("routerarp|") - .append(sn.getId().getValue()) - .append("|") - .append(ikey) - .append("|") - .append(l3Id) - .toString()); - if (visit(flowMap, flowId.getValue())) { - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(ethernetMatch(null, null, ARP)) - .setLayer3Match(new ArpMatchBuilder() - .setArpOp(Integer.valueOf(1)) - .setArpTargetTransportAddress(new Ipv4Prefix(ikey+"/32")) - .build()); - addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, - Long.valueOf(l3Id))); - BigInteger routerMac = - new BigInteger(1, bytesFromHexString(ROUTER_MAC - .getValue())); - FlowBuilder flowb = base() - .setPriority(150) - .setId(flowId) - .setMatch(mb.build()) - .setInstructions(instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), - setDlSrcAction(ROUTER_MAC), - nxLoadArpOpAction(BigInteger.valueOf(2L)), - nxMoveArpShaToArpThaAction(), - nxLoadArpShaAction(routerMac), - nxMoveArpSpaToArpTpaAction(), - nxLoadArpSpaAction(ikey), - outputAction(new NodeConnectorId(nodeId.getValue() + ":INPORT"))))); - writeFlow(t, tiid, flowb.build()); - } + if (ctx.getDataBroker() == null) { + return null; + } + + MacAddress defaultMacAddress = ROUTER_MAC; + + EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId()); + InstanceIdentifier endpointsIid = InstanceIdentifier.builder(Endpoints.class) + .child(EndpointL3.class, l3Key) + .build(); + ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction(); + + Optional r; + try { + r = t.read(LogicalDatastoreType.OPERATIONAL, endpointsIid).get(); + if (!r.isPresent()) + return defaultMacAddress; + EndpointL3 epL3 = r.get(); + if (epL3.getMacAddress() == null) { + return defaultMacAddress; } else { - LOG.warn("IPv6 virtual router {} for subnet {} not supported", - sn.getVirtualRouterIp(), sn.getId().getValue()); + return epL3.getMacAddress(); + } + } catch (Exception e) { + LOG.error("Error reading EndpointL3 {}.{}", l3c, ipAddress, e); + return null; + } + } + + private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) { + L3Context l3c = ctx.getPolicyResolver().getTenant(tenantId).resolveL3Context(sn.getId()); + return l3c; + } + + private Flow createRouterArpFlow(TenantId tenantId, NodeId nodeId, Subnet sn, int l3Id) { + if (sn == null || sn.getVirtualRouterIp() == null) { + LOG.trace("Didn't create routerArpFlow since either subnet or subnet virtual router was null"); + return null; + } + /* + * TODO: Li alagalah: This should be new Yang "gateways" list as well, + * that expresses the gateway and prefixes it is interface for. Should + * also check for external. + */ + if (sn.getVirtualRouterIp().getIpv4Address() != null) { + String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue(); + + L3Context l3c = getL3ContextForSubnet(tenantId, sn); + if (l3c == null) { + LOG.error("No L3 Context found associated with subnet {}", sn.getId()); + } + + MacAddress routerMac = routerPortMac(l3c, sn.getVirtualRouterIp()); + if (routerMac == null) { + return null; } + + BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue())); + + FlowId flowId = new FlowId(new StringBuffer().append("routerarp|") + .append(sn.getId().getValue()) + .append("|") + .append(ikey) + .append("|") + .append(l3Id) + .toString()); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match( + new ArpMatchBuilder().setArpOp(Integer.valueOf(1)) + .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32")) + .build()); + addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id))); + + FlowBuilder flowb = base().setPriority(150) + .setId(flowId) + .setMatch(mb.build()) + .setInstructions( + instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac), + nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(), + nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(), + nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue() + + ":INPORT"))))); + return flowb.build(); + } else { + LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId() + .getValue()); + return null; } + } - private void syncEP(ReadWriteTransaction t, - InstanceIdentifier
tiid, - Map flowMap, - NodeId nodeId, PolicyInfo policyInfo, - Endpoint e, OfOverlayContext ofc, - IndexedTenant tenant, EgKey key) - throws Exception { + private Flow createLocalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) { + + // TODO Li alagalah - refactor common code but keep simple method ArrayList instructions = new ArrayList<>(); - ArrayList l3instructions = new ArrayList<>(); List applyActions = new ArrayList<>(); - List l3ApplyActions = new ArrayList<>(); int order = 0; - EndpointGroup eg = tenant.getEndpointGroup(e.getEndpointGroup()); - L3Context l3c = tenant.resolveL3Context(eg.getNetworkDomain()); - L2BridgeDomain bd = tenant.resolveL2BridgeDomain(eg.getNetworkDomain()); - - int egId = 0, bdId = 0, l3Id = 0, cgId = 0; - - egId = ctx.getPolicyManager().getContextOrdinal(e.getTenant(), - e.getEndpointGroup()); - if (bd != null) - bdId = ctx.getPolicyManager().getContextOrdinal(e.getTenant(), - bd.getId()); - if (l3c != null) - l3Id = ctx.getPolicyManager().getContextOrdinal(e.getTenant(), - l3c.getId()); - - List conds = ctx.getEndpointManager().getCondsForEndpoint(e); - ConditionGroup cg = - policyInfo.getEgCondGroup(new EgKey(e.getTenant(), - e.getEndpointGroup()), - conds); - cgId = ctx.getPolicyManager().getCondGroupOrdinal(cg); - Action setdEPG = nxLoadRegAction(NxmNxReg2.class, - BigInteger.valueOf(egId)); - Action setdCG = nxLoadRegAction(NxmNxReg3.class, - BigInteger.valueOf(cgId)); + + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId())); Action setNextHop; String nextHop; + + // BEGIN L2 LOCAL + nextHop = ofc.getNodeConnectorId().getValue(); + + long portNum; + try { + portNum = getOfPortNum(ofc.getNodeConnectorId()); + } catch (NumberFormatException ex) { + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); + return null; + } + + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); + + // END L2 LOCAL + + order += 1; + applyActions.add(setdEPG); + applyActions.add(setdCG); + applyActions.add(setNextHop); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); + instructions.add(applyActionsIns); + + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns((short) (getTableId() + 1))) + .build(); + instructions.add(gotoTable); + + FlowId flowid = new FlowId(new StringBuilder().append(epFwdCtxOrds.getBdId()) + .append("|l2|") + .append(ep.getMacAddress().getValue()) + .append("|") + .append(nextHop) + .toString()); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null)); + addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId()))); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(50)) + .setMatch(mb.build()) + .setInstructions(new InstructionsBuilder().setInstruction(instructions).build()); + return flowb.build(); + } + + private void syncEP(FlowMap flowMap, NodeId nodeId, PolicyInfo policyInfo, Endpoint srcEp, Endpoint destEp) + throws Exception { + + // TODO: Conditions messed up, but for now, send policyInfo until this + // is fixed. + EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, destEp); + EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, srcEp); + + if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) { + LOG.trace("Didn't process endpoint due to either tenant, or EPG(s) being null", destEp.getKey()); + return; + } + OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class); + + // //////////////////////////////////////////////////////////////////////////////////////// + /* + * NOT HANDLING EXTERNALS TODO: alagalah Li: External Gateway + * functionality needed here. + */ if (LocationType.External.equals(ofc.getLocationType())) { // XXX - TODO - perform NAT and send to the external network - nextHop = "external"; + // TODO: Use case neutron gateway interface LOG.warn("External endpoints not yet supported"); return; - } else { - Action setDlSrc = setDlSrcAction(ROUTER_MAC); - Action decTtl = decNwTtlAction(); - - if (Objects.equals(ofc.getNodeId(), nodeId)) { - // this is a local endpoint; send to the approppriate local - // port - nextHop = ofc.getNodeConnectorId().getValue(); - - long portNum; - try { - portNum = getOfPortNum(ofc.getNodeConnectorId()); - } catch (NumberFormatException ex) { - LOG.warn("Could not parse port number {}", - ofc.getNodeConnectorId(), ex); - return; - } + } + + /* + * Only care about subnets for L3, but fetch them before loop. We need + * the local subnets for setting SRC MAC for routing. All Routing is now + * done locally! YAY! Instead of being shovelled L2 style across network + * ala Helium. + */ + List localSubnets = getLocalSubnets(nodeId); + if (localSubnets == null) { + LOG.error("No subnets could be found locally for node: {}", nodeId); + return; + } + + if (Objects.equals(ofc.getNodeId(), nodeId)) { + // this is a local endpoint; send to the approppriate local + // port - setNextHop = nxLoadRegAction(NxmNxReg7.class, - BigInteger.valueOf(portNum)); + if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) { + flowMap.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc)); + } + // TODO Li alagalah: Need to move to EndpointL3 for L3 processing. + // The Endpoint conflation must end! + if (destEp.getL3Address() == null) { + LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey()); + return; + } - Action setDlDst = setDlDstAction(e.getMacAddress()); - l3ApplyActions.add(setDlSrc); - l3ApplyActions.add(setDlDst); - l3ApplyActions.add(decTtl); - order +=1; + for (L3Address l3a : destEp.getL3Address()) { + if (l3a.getIpAddress() == null || l3a.getL3Context() == null) { + LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}", + destEp.getL3Address()); + continue; + } else { + for (Subnet localSubnet : localSubnets) { + Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet); + if (flow != null) { + flowMap.writeFlow(nodeId, TABLE_ID, flow); + } else { + LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(), + localSubnet.getIpPrefix().getValue()); + } + } + } + } + } else { + // this endpoint is on a different switch; send to the + // appropriate tunnel + if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) { + Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc); + if (remoteL2Flow != null) { + flowMap.writeFlow(nodeId, TABLE_ID, remoteL2Flow); + } } else { - // this endpoint is on a different switch; send to the - // appropriate tunnel - - IpAddress tunDst = - ctx.getSwitchManager().getTunnelIP(ofc.getNodeId()); - NodeConnectorId tunPort = - ctx.getSwitchManager().getTunnelPort(nodeId); - if (tunDst == null) return; - if (tunPort == null) return; - - Action tundstAction; - - if (tunDst.getIpv4Address() != null) { - nextHop = tunDst.getIpv4Address().getValue(); - tundstAction = nxLoadTunIPv4Action(nextHop, false); - } else if (tunDst.getIpv6Address() != null) { - // nextHop = tunDst.getIpv6Address().getValue(); - LOG.error("IPv6 tunnel destination {} for {} not supported", - tunDst.getIpv6Address().getValue(), - ofc.getNodeId()); - return; + LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}", + srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId()); + } + + // TODO Li alagalah: Need to move to EndpointL3 for L3 processing. + // The Endpoint conflation must end! + if (destEp.getL3Address() == null) { + LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey()); + return; + } + for (L3Address l3a : destEp.getL3Address()) { + if (l3a.getIpAddress() == null || l3a.getL3Context() == null) { + LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}", + destEp.getL3Address()); + continue; } else { - // this shouldn't happen - LOG.error("Tunnel IP for {} invalid", ofc.getNodeId()); - return; + for (Subnet localSubnet : localSubnets) { + Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds, + destEpFwdCtxOrds, ofc, localSubnet); + if (remoteL3Flow != null) { + flowMap.writeFlow(nodeId, TABLE_ID, remoteL3Flow); + } else { + LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(), + localSubnet.getIpPrefix().getValue()); + } + } } + } + } // remote (tunnel) + // } - long portNum; - try { - portNum = getOfPortNum(tunPort); - } catch (NumberFormatException ex) { - LOG.warn("Could not parse port number {}", - ofc.getNodeConnectorId(), ex); - return; - } + } - setNextHop = nxLoadRegAction(NxmNxReg7.class, - BigInteger.valueOf(portNum)); - Action tunIdAction = - nxMoveRegTunIdAction(NxmNxReg0.class, false); + /* + * ################################## DestMapper Flow methods + * ################################## + */ + private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds, + OfOverlayContext ofc, Subnet srcSubnet) { - applyActions.add(tunIdAction); - applyActions.add(tundstAction); - l3ApplyActions.add(setDlSrc); - l3ApplyActions.add(decTtl); - order +=1; + // TODO Li alagalah - refactor common code but keep simple method + + Subnet destSubnet = null; + HashSet subnets = getSubnets(destEp.getTenant()); + if (subnets == null) { + LOG.trace("No subnets in tenant {}", destL3Address.getIpAddress()); + return null; + } + NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp); + for (Subnet subnet : subnets) { + // TODO Li alagalah add IPv6 support + if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) { + destSubnet = subnet; + break; } } + if (destSubnet == null) { + LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress()); + return null; + } + + if (destSubnet.getVirtualRouterIp() == null) { + LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(), + destL3Address.getKey()); + return null; + } + + if (srcSubnet.getVirtualRouterIp() == null) { + LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix()); + return null; + } + L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet); + if (destL3c == null || destL3c.getId() == null) { + LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId()); + return null; + } + L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet); + if (srcL3c == null || srcL3c.getId() == null) { + LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId()); + return null; + } + + if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) { + LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId() + .getValue(), destL3c.getId().getValue()); + return null; + } + + MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp()); + MacAddress epDestMac = destEp.getMacAddress(); + MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp()); + + if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) { + // This is our final destination, so match on actual EP mac. + matcherMac = epDestMac; + } + + ArrayList l3instructions = new ArrayList<>(); + List applyActions = new ArrayList<>(); + List l3ApplyActions = new ArrayList<>(); + + int order = 0; + + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId())); + Action setNextHop; + String nextHop; + + // BEGIN L3 LOCAL + nextHop = ofc.getNodeConnectorId().getValue(); + + long portNum; + try { + portNum = getOfPortNum(ofc.getNodeConnectorId()); + } catch (NumberFormatException ex) { + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); + return null; + } + + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); + // END L3 LOCAL + + // Lets not re-write the srcMac if its local. + if (!(matcherMac.getValue().equals(epDestMac.getValue()))) { + Action setDlSrc = setDlSrcAction(destSubnetGatewayMac); + l3ApplyActions.add(setDlSrc); + } + + Action setDlDst = setDlDstAction(epDestMac); + l3ApplyActions.add(setDlDst); + + Action decTtl = decNwTtlAction(); + l3ApplyActions.add(decTtl); + + order += 1; applyActions.add(setdEPG); applyActions.add(setdCG); applyActions.add(setNextHop); - Instruction applyActionsIns = new InstructionBuilder() - .setOrder(order++) + + applyActions.addAll(l3ApplyActions); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); + + l3instructions.add(applyActionsIns); + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns((short) (getTableId() + 1))) + .build(); + l3instructions.add(gotoTable); + Layer3Match m = null; + Long etherType = null; + String ikey = null; + if (destL3Address.getIpAddress().getIpv4Address() != null) { + ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32"; + etherType = IPv4; + m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build(); + } else if (destL3Address.getIpAddress().getIpv6Address() != null) { + ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128"; + etherType = IPv6; + m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build(); + } else { + LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString()); + return null; + } + + FlowId flowid = new FlowId(new StringBuilder().append(Integer.toString(epFwdCtxOrds.getL3Id())) + .append("|l3|") + .append(ikey) + .append("|") + .append(Integer.toString(epFwdCtxOrds.getEpgId())) + .append("|") + .append(Integer.toString(epFwdCtxOrds.getCgId())) + .append("|") + .append(matcherMac) + .append("|") + .append(destSubnetGatewayMac) + .append("|") + .append(nextHop) + .toString()); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType)) + .setLayer3Match(m); + addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id()))); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(132)) + .setMatch(mb.build()) + .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build()); + return flowb.build(); + } + + private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds, + EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) { + + // TODO Li alagalah - refactor common code but keep simple method + + // this endpoint is on a different switch; send to the + // appropriate tunnel + + ArrayList instructions = new ArrayList<>(); + List applyActions = new ArrayList<>(); + + int order = 0; + + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId())); + Action setNextHop; + String nextHop; + + // BEGIN TUNNEL HANDLING + IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class); + NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class); + if (tunDst == null) { + LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep); + return null; + } + if (tunPort == null) { + LOG.warn("Failed to get Tunnel Port for NodeId {} with EP {}", nodeId, ep); + return null; + } + + Action tundstAction; + + if (tunDst.getIpv4Address() != null) { + nextHop = tunDst.getIpv4Address().getValue(); + tundstAction = nxLoadTunIPv4Action(nextHop, false); + } else if (tunDst.getIpv6Address() != null) { + // nextHop = tunDst.getIpv6Address().getValue(); + LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(), + ofc.getNodeId()); + return null; + } else { + // this shouldn't happen + LOG.error("Tunnel IP for {} invalid", ofc.getNodeId()); + return null; + } + + long portNum; + try { + portNum = getOfPortNum(tunPort); + } catch (NumberFormatException ex) { + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); + return null; + } + + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); + applyActions.add(tundstAction); + // END TUNNEL + + order += 1; + applyActions.add(setdEPG); + applyActions.add(setdCG); + applyActions.add(setNextHop); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) .build(); instructions.add(applyActionsIns); - applyActions.addAll(l3ApplyActions); - applyActionsIns = new InstructionBuilder() - .setOrder(order++) + applyActionsIns = new InstructionBuilder().setOrder(order++) .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) .build(); - l3instructions.add(applyActionsIns); - Instruction gotoTable = new InstructionBuilder() - .setOrder(order++) - .setInstruction(gotoTableIns((short)(getTableId()+1))) + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns((short) (getTableId() + 1))) .build(); instructions.add(gotoTable); - l3instructions.add(gotoTable); - FlowId flowid = new FlowId(new StringBuilder() - .append(bdId) + FlowId flowid = new FlowId(new StringBuilder().append(destEpFwdCtxOrds.getBdId()) .append("|l2|") - .append(e.getMacAddress().getValue()) + .append(ep.getMacAddress().getValue()) + .append("|") + .append(srcEpFwdCtxOrds.getBdId()) .append("|") .append(nextHop) .toString()); - if (visit(flowMap, flowid.getValue())) { - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(ethernetMatch(null, - e.getMacAddress(), - null)); - addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(bdId))); - FlowBuilder flowb = base() - .setId(flowid) - .setPriority(Integer.valueOf(50)) - .setMatch(mb.build()) - .setInstructions(new InstructionsBuilder() - .setInstruction(instructions) - .build()); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null)); + addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId()))); + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(50)) + .setMatch(mb.build()) + .setInstructions(new InstructionsBuilder().setInstruction(instructions).build()); + + return flowb.build(); + } + + private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId, + EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc, + Subnet srcSubnet) { - writeFlow(t, tiid, flowb.build()); + // TODO Li alagalah - refactor common code but keep simple method + + // this endpoint is on a different switch; send to the + // appropriate tunnel + Subnet destSubnet = null; + HashSet subnets = getSubnets(destEp.getTenant()); + if (subnets == null) { + LOG.trace("No subnets in tenant {}", destL3Address.getIpAddress()); + return null; + } + NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp); + for (Subnet subnet : subnets) { + // TODO Li alagalah add IPv6 support + if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) { + destSubnet = subnet; + break; + } + } + if (destSubnet == null) { + LOG.info("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress()); + return null; } - if (e.getL3Address() == null) return; - for (L3Address l3a : e.getL3Address()) { - if (l3a.getIpAddress() == null || l3a.getL3Context() == null) - continue; - Layer3Match m = null; - Long etherType = null; - String ikey = null; - if (l3a.getIpAddress().getIpv4Address() != null) { - ikey = l3a.getIpAddress().getIpv4Address().getValue() + "/32"; - etherType = IPv4; - m = new Ipv4MatchBuilder() - .setIpv4Destination(new Ipv4Prefix(ikey)) - .build(); - } else if (l3a.getIpAddress().getIpv6Address() != null) { - ikey = l3a.getIpAddress().getIpv6Address().getValue() + "/128"; - etherType = IPv6; - m = new Ipv6MatchBuilder() - .setIpv6Destination(new Ipv6Prefix(ikey)) - .build(); - } else - continue; - flowid = new FlowId(new StringBuilder() - .append(l3a.getL3Context().getValue()) - .append("|l3|") - .append(ikey) - .append("|") - .append(nextHop) - .toString()); - if (visit(flowMap, flowid.getValue())) { - MatchBuilder mb = new MatchBuilder() - .setEthernetMatch(ethernetMatch(null, - ROUTER_MAC, - etherType)) - .setLayer3Match(m); - addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, - Long.valueOf(l3Id))); - FlowBuilder flowb = base() - .setId(flowid) - .setPriority(Integer.valueOf(132)) - .setMatch(mb.build()) - .setInstructions(new InstructionsBuilder() - .setInstruction(l3instructions) - .build()); + if (destSubnet.getVirtualRouterIp() == null) { + LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(), + destL3Address.getKey()); + return null; + } + + if (srcSubnet.getVirtualRouterIp() == null) { + LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix()); + return null; + } + L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet); + if (destL3c == null || destL3c.getId() == null) { + LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId()); + return null; + } + L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet); + if (srcL3c == null || srcL3c.getId() == null) { + LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId()); + return null; + } + + if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) { + LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId() + .getValue(), destL3c.getId().getValue()); + return null; + } - writeFlow(t, tiid, flowb.build()); + MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp()); + MacAddress epDestMac = destEp.getMacAddress(); + MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp()); + + ArrayList l3instructions = new ArrayList<>(); + List applyActions = new ArrayList<>(); + List l3ApplyActions = new ArrayList<>(); + + int order = 0; + + Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId())); + Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId())); + Action setNextHop; + String nextHop; + + // BEGIN TUNNEL HANDLING + IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class); + NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class); + if (tunDst == null) { + LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address); + return null; + } + if (tunPort == null) { + LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, destL3Address); + return null; + } + + Action tundstAction; + + if (tunDst.getIpv4Address() != null) { + nextHop = tunDst.getIpv4Address().getValue(); + tundstAction = nxLoadTunIPv4Action(nextHop, false); + } else if (tunDst.getIpv6Address() != null) { + // nextHop = tunDst.getIpv6Address().getValue(); + LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(), + ofc.getNodeId()); + return null; + } else { + // this shouldn't happen + LOG.error("Tunnel IP for {} invalid", ofc.getNodeId()); + return null; + } + + long portNum; + try { + portNum = getOfPortNum(tunPort); + } catch (NumberFormatException ex) { + LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex); + return null; + } + + setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum)); + applyActions.add(tundstAction); + // END TUNNEL + + order += 1; + applyActions.add(setdEPG); + applyActions.add(setdCG); + applyActions.add(setNextHop); + + Action setDlSrc = setDlSrcAction(destSubnetGatewayMac); + l3ApplyActions.add(setDlSrc); + + Action setDlDst = setDlDstAction(epDestMac); + l3ApplyActions.add(setDlDst); + + Action decTtl = decNwTtlAction(); + l3ApplyActions.add(decTtl); + + applyActions.addAll(l3ApplyActions); + Instruction applyActionsIns = new InstructionBuilder().setOrder(order++) + .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()]))) + .build(); + + l3instructions.add(applyActionsIns); + Instruction gotoTable = new InstructionBuilder().setOrder(order++) + .setInstruction(gotoTableIns((short) (getTableId() + 1))) + .build(); + l3instructions.add(gotoTable); + Layer3Match m = null; + Long etherType = null; + String ikey = null; + if (destL3Address.getIpAddress().getIpv4Address() != null) { + ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32"; + etherType = IPv4; + m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build(); + } else if (destL3Address.getIpAddress().getIpv6Address() != null) { + ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128"; + etherType = IPv6; + m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build(); + } else { + LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString()); + return null; + } + + FlowId flowid = new FlowId(new StringBuilder().append(Integer.toString(destEpFwdCtxOrds.getL3Id())) + .append("|l3|") + .append(ikey) + .append("|") + .append(matcherMac) + .append("|") + .append(destSubnetGatewayMac) + .append("|") + .append(srcEpFwdCtxOrds.getL3Id()) + .append("|") + .append(nextHop) + .toString()); + MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType)) + .setLayer3Match(m); + addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id()))); + + FlowBuilder flowb = base().setId(flowid) + .setPriority(Integer.valueOf(132)) + .setMatch(mb.build()) + .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build()); + return flowb.build(); + } + + private NetworkDomainId getEPNetworkContainment(Endpoint endpoint) { + if (endpoint.getNetworkContainment() != null) { + return endpoint.getNetworkContainment(); + } else { + /* + * TODO: Be alagalah: Endpoint Refactor: This should be set on input + * which we can't do because of the backwards way endpoints were + * "architected". + */ + return ctx.getPolicyResolver() + .getTenant(endpoint.getTenant()) + .getEndpointGroup(endpoint.getEndpointGroup()) + .getNetworkDomain(); + } + } + + private HashSet getSubnets(final TenantId tenantId) { + + // if (subnetsByTenant.get(tenantId) != null) { + // return subnetsByTenant.get(tenantId); + // } + + if (ctx.getDataBroker() == null) { + return null; + } + + ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction(); + InstanceIdentifier tiid = TenantUtils.tenantIid(tenantId); + Optional tenantInfo; + try { + tenantInfo = t.read(LogicalDatastoreType.CONFIGURATION, tiid).get(); + } catch (Exception e) { + LOG.error("Could not read Tenant {}", tenantId, e); + return null; + } + + HashSet subnets = new HashSet(); + + if (!tenantInfo.isPresent()) { + LOG.warn("Tenant {} not found", tenantId); + return null; + } + + subnets.addAll(tenantInfo.get().getSubnet()); + // subnetsByTenant.put(tenantId, subnets); + return subnets; + } + + // Need a method to get subnets for EPs attached to the node locally + // to set the source Mac address for the router interface. + private List getLocalSubnets(NodeId nodeId) { + Collection endpointsForNode = ctx.getEndpointManager().getEndpointsForNode(nodeId); + + List localSubnets = new ArrayList(); + + for (Endpoint endpoint : endpointsForNode) { + HashSet subnets = getSubnets(endpoint.getTenant()); + if (subnets == null) { + LOG.error("No local subnets."); + return null; + } + NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint); + for (Subnet subnet : subnets) { + if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) { + localSubnets.add(subnet); + } } } + return localSubnets; + } + + /** + * Reads data from datastore as synchronous call. + * + * @return {@link Optional#isPresent()} is {@code true} if reading was + * successful and data exists in datastore; {@link Optional#isPresent()} is + * {@code false} otherwise + */ + public static Optional readFromDs(LogicalDatastoreType store, InstanceIdentifier path, + ReadTransaction rTx) { + CheckedFuture, ReadFailedException> resultFuture = rTx.read(store, path); + try { + return resultFuture.checkedGet(); + } catch (ReadFailedException e) { + LOG.warn("Read failed from DS.", e); + return Optional.absent(); + } } static byte[] bytesFromHexString(String values) {