renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java

   1 /*
   2  * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow;
  10
  11 import com.google.common.base.Optional;
  12 import com.google.common.base.Strings;
  13 import com.google.common.collect.HashMultimap;
  14 import com.google.common.collect.SetMultimap;
  15 import com.google.common.collect.Sets;
  16 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  17 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  18 import org.opendaylight.groupbasedpolicy.endpoint.EpKey;
  19 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext;
  20 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.PolicyManager.FlowMap;
  21 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch;
  22 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals;
  23 import org.opendaylight.groupbasedpolicy.resolver.EgKey;
  24 import org.opendaylight.groupbasedpolicy.resolver.IndexedTenant;
  25 import org.opendaylight.groupbasedpolicy.resolver.PolicyInfo;
  26 import org.opendaylight.groupbasedpolicy.resolver.TenantUtils;
  27 import org.opendaylight.groupbasedpolicy.util.IidFactory;
  28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  29 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
  30 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix;
  31 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.l3.prefix.fields.EndpointL3Gateways;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Prefix;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.EndpointLocation.LocationType;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.L3Context;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.Subnet;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.ethernet.match.fields.EthernetDestinationBuilder;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatchBuilder;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.Layer3Match;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
  74 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  75 import org.slf4j.Logger;
  76 import org.slf4j.LoggerFactory;
  77
  78 import java.math.BigInteger;
  79 import java.util.ArrayList;
  80 import java.util.Collection;
  81 import java.util.Collections;
  82 import java.util.HashMap;
  83 import java.util.HashSet;
  84 import java.util.List;
  85 import java.util.Map;
  86 import java.util.Map.Entry;
  87 import java.util.Objects;
  88 import java.util.Set;
  89
  90 import static com.google.common.base.Preconditions.checkNotNull;
  91 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP;
  92 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4;
  93 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6;
  94 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch;
  95 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns;
  96 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.createNodePath;
  97 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction;
  98 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch;
  99 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum;
 100 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.gotoTableIns;
 101 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.groupAction;
 102 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.instructions;
 103 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpOpAction;
 104 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpShaAction;
 105 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction;
 106 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction;
 107 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action;
 108 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction;
 109 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction;
 110 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction;
 111 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction;
 112 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction;
 113 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction;
 114 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction;
 115 import static org.opendaylight.groupbasedpolicy.util.DataStoreHelper.readFromDs;
 116
 117 /**
 118  * Manage the table that maps the destination address to the next hop for the
 119  * path as well as applies any relevant routing transformations.
 120  */
 121 public class DestinationMapper extends FlowTable {
 122
 123     protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class);
 124
 125     // TODO Li alagalah: Improve UT coverage for this class.
 126
 127     // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows
 128     // This ensures we have the appropriate network-containment'
 129
 130     public static short TABLE_ID;
 131     /**
 132      * This is the MAC address of the magical router in the sky
 133      */
 134     public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5");
 135     public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00");
 136     public static final Integer BASE_L3_PRIORITY = 100;
 137
 138     public DestinationMapper(OfContext ctx, short tableId) {
 139         super(ctx);
 140         this.TABLE_ID = tableId;
 141     }
 142
 143     Map<TenantId, HashSet<Subnet>> subnetsByTenant = new HashMap<TenantId, HashSet<Subnet>>();
 144
 145     @Override
 146     public short getTableId() {
 147         return TABLE_ID;
 148     }
 149
 150     @Override
 151     public void sync(NodeId nodeId, PolicyInfo policyInfo, FlowMap flowMap) throws Exception {
 152
 153         TenantId currentTenant = null;
 154
 155         flowMap.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null, TABLE_ID));
 156
 157         SetMultimap<EpKey, EpKey> visitedEps = HashMultimap.create();
 158         Set<EndpointFwdCtxOrdinals> epOrdSet = new HashSet<>();
 159
 160         for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) {
 161             Set<EndpointGroupId> srcEpgIds = new HashSet<>();
 162             if (srcEp.getEndpointGroup() != null)
 163                 srcEpgIds.add(srcEp.getEndpointGroup());
 164             if (srcEp.getEndpointGroups() != null)
 165                 srcEpgIds.addAll(srcEp.getEndpointGroups());
 166
 167             for (EndpointGroupId epgId : srcEpgIds) {
 168                 EgKey epg = new EgKey(srcEp.getTenant(), epgId);
 169                 Set<EgKey> peers = Sets.union(Collections.singleton(epg), policyInfo.getPeers(epg));
 170                 for (EgKey peer : peers) {
 171                     for (Endpoint peerEp : ctx.getEndpointManager().getEndpointsForGroup(peer)) {
 172                         currentTenant = peerEp.getTenant();
 173                         subnetsByTenant.put(currentTenant, getSubnets(currentTenant));
 174                         EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress());
 175                         EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress());
 176
 177                         if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) {
 178                             continue;
 179                         }
 180                         syncEP(flowMap, nodeId, policyInfo, srcEp, peerEp);
 181                         visitedEps.put(srcEpKey, peerEpKey);
 182
 183                         // Process subnets and flood-domains for epPeer
 184                         EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo,
 185                                 peerEp);
 186                         epOrdSet.add(epOrds);
 187                     }
 188                 }
 189             }
 190         }
 191
 192         for (Entry<TenantId, HashSet<Subnet>> subnetEntry : subnetsByTenant.entrySet()) {
 193             if (subnetEntry.getValue() == null) {
 194                 LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey());
 195                 continue;
 196             }
 197             currentTenant = subnetEntry.getKey();
 198             for (Subnet sn : subnetEntry.getValue()) {
 199                 L3Context l3c = getL3ContextForSubnet(currentTenant, sn);
 200                 Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn,
 201                         OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId()));
 202                 if (arpFlow != null) {
 203                     flowMap.writeFlow(nodeId, TABLE_ID, arpFlow);
 204                 } else {
 205                     LOG.debug(
 206                             "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .",
 207                             sn.getIpPrefix().getValue());
 208                 }
 209             }
 210         }
 211
 212         // Write broadcast flows per flood domain.
 213         for (EndpointFwdCtxOrdinals epOrd : epOrdSet) {
 214             if (groupExists(nodeId, epOrd.getFdId())) {
 215                 flowMap.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd));
 216             }
 217         }
 218
 219         // L3 Prefix Endpoint handling
 220         Collection<EndpointL3Prefix> prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(currentTenant);
 221         if (prefixEps != null) {
 222             LOG.trace("DestinationMapper - Processing L3PrefixEndpoints");
 223             for (EndpointL3Prefix prefixEp : prefixEps) {
 224                 List<Subnet> localSubnets = getLocalSubnets(nodeId);
 225                 if (localSubnets == null) {
 226                     continue;
 227                 }
 228                 for (Subnet localSubnet: localSubnets) {
 229                     Flow prefixFlow = createL3PrefixFlow(prefixEp, policyInfo, nodeId, localSubnet);
 230                     if (prefixFlow != null) {
 231                         flowMap.writeFlow(nodeId, TABLE_ID, prefixFlow);
 232                         LOG.trace("Wrote L3Prefix flow");
 233                     }
 234                 }
 235             }
 236         }
 237
 238     }
 239
 240     // set up next-hop destinations for all the endpoints in the endpoint
 241     // group on the node
 242
 243     private Flow createL3PrefixFlow(EndpointL3Prefix prefixEp, PolicyInfo policyInfo, NodeId nodeId, Subnet subnet) throws Exception {
 244         /*
 245          * Priority: 100+lengthprefix
 246          * Match: prefix, l3c, "mac address of router" ?
 247          * Action:
 248          * - set Reg2, Reg3 for L3Ep by L2Ep ?
 249          * - if external,
 250          * - Reg7: use switch location external port else punt for now
 251          * - if internal
 252          * - Reg7: grab L2Ep from L3Ep and use its location info
 253          * - goto_table: POLENF (will check there for external on EP)
 254          */
 255
 256         ReadOnlyTransaction rTx = ctx.getDataBroker().newReadOnlyTransaction();
 257         // TODO Bug #3440 Target: Be - should support for more than first gateway.
 258         EndpointL3Gateways l3Gateway = prefixEp.getEndpointL3Gateways().get(0);
 259         Optional<EndpointL3> optL3Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 260                 IidFactory.l3EndpointIid(l3Gateway.getL3Context(), l3Gateway.getIpAddress()), rTx);
 261         if (!optL3Ep.isPresent()) {
 262             LOG.error("createL3PrefixFlow - L3Endpoint gateway {} for L3Prefix {} not found.", l3Gateway, prefixEp);
 263             return null;
 264         }
 265         EndpointL3 l3Ep = optL3Ep.get();
 266         if (l3Ep.getL2Context() == null || l3Ep.getMacAddress() == null) {
 267             LOG.debug("L3 endpoint representing L3 gateway does not contain L2-context or MAC address. {}", l3Ep);
 268             return null;
 269         }
 270         Optional<Endpoint> optL2Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 271                 IidFactory.endpointIid(l3Ep.getL2Context(), l3Ep.getMacAddress()), rTx);
 272         if (!optL2Ep.isPresent()) {
 273             LOG.error("createL3PrefixFlow - L2Endpoint for L3Gateway {} not found.", l3Ep);
 274             return null;
 275         }
 276         Endpoint l2Ep = optL2Ep.get();
 277         EndpointFwdCtxOrdinals epFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, l2Ep);
 278
 279         NetworkDomainId epNetworkContainment = getEPNetworkContainment(l2Ep);
 280
 281         MacAddress epDestMac = l2Ep.getMacAddress();
 282         MacAddress destSubnetGatewayMac = l2Ep.getMacAddress();
 283         L3Context destL3c = getL3ContextForSubnet(prefixEp.getTenant(), subnet);
 284         if (destL3c == null || destL3c.getId() == null) {
 285             LOG.error("No L3 Context found associated with subnet {}", subnet.getId());
 286             return null;
 287         }
 288
 289         MacAddress matcherMac = routerPortMac(destL3c, subnet.getVirtualRouterIp());
 290
 291         ArrayList<Instruction> l3instructions = new ArrayList<>();
 292         List<Action> applyActions = new ArrayList<>();
 293         List<Action> l3ApplyActions = new ArrayList<>();
 294
 295         int order = 0;
 296
 297         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 298         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 299         Action setNextHop;
 300         String nextHop=null;
 301
 302         OfOverlayContext ofc = l2Ep.getAugmentation(OfOverlayContext.class);
 303         LocationType location;
 304
 305         if (ofc != null && ofc.getLocationType() != null) {
 306             location = ofc.getLocationType();
 307         } else if (ofc != null) {
 308             // Augmentation, but using default location
 309             location = LocationType.Internal;
 310         } else {
 311             LOG.info("createL3PrefixFlow - Endpoint {} had no augmentation.", l2Ep);
 312             return null;
 313         }
 314
 315         long portNum = -1;
 316
 317         if (location.equals(LocationType.Internal)) {
 318             checkNotNull(ofc.getNodeConnectorId());
 319             nextHop = ofc.getNodeConnectorId().getValue();
 320             try {
 321                 portNum = getOfPortNum(ofc.getNodeConnectorId());
 322             } catch (NumberFormatException ex) {
 323                 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 324                 return null;
 325             }
 326
 327         } else {
 328             // External
 329             Set<NodeConnectorId> externalPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
 330             checkNotNull(externalPorts);
 331             for (NodeConnectorId externalPort : externalPorts) {
 332                 // TODO Bug #3440 Target: Be - should support for more than first external port.
 333                 //TODO Bug 3546 - Difficult: External port is unrelated to Tenant, L3C, L2BD..
 334                 nextHop = externalPort.getValue();
 335                 try {
 336                     portNum = getOfPortNum(externalPort);
 337                 } catch (NumberFormatException ex) {
 338                     LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 339                     return null;
 340                 }
 341                 continue;
 342             }
 343         }
 344
 345         if (Strings.isNullOrEmpty(nextHop)
 346                 || portNum == -1) {
 347             LOG.error("createL3Prefix - Cannot find nodeConnectorId for {} for Prefix: ", l2Ep, prefixEp);
 348             return null;
 349         }
 350         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 351
 352         Action setDlDst = setDlDstAction(epDestMac);
 353         l3ApplyActions.add(setDlDst);
 354
 355         Action decTtl = decNwTtlAction();
 356         l3ApplyActions.add(decTtl);
 357
 358         order += 1;
 359         applyActions.add(setdEPG);
 360         applyActions.add(setdCG);
 361         applyActions.add(setNextHop);
 362
 363         applyActions.addAll(l3ApplyActions);
 364         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 365             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 366             .build();
 367
 368         l3instructions.add(applyActionsIns);
 369         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 370             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 371             .build();
 372         l3instructions.add(gotoTable);
 373
 374         Layer3Match m = null;
 375         Long etherType = null;
 376         String ikey = null;
 377         Integer prefixLength=0;
 378         if (prefixEp.getIpPrefix().getIpv4Prefix() != null) {
 379             ikey = prefixEp.getIpPrefix().getIpv4Prefix().getValue();
 380             etherType = IPv4;
 381             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv4Prefix().getValue().split("/")[1]);
 382             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 383         } else if (prefixEp.getIpPrefix().getIpv6Prefix() != null) {
 384             ikey = prefixEp.getIpPrefix().getIpv6Prefix().getValue();
 385             etherType = IPv6;
 386             /*
 387              *  This will result in flows with priority between 100-228, but since its matching on IPv6 prefix as well
 388              *  this shouldn't pose and issue, as the priority is more important within the address space of the matcher,
 389              *  even though technically flows are processed in priority order.
 390              */
 391
 392             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv6Prefix().getValue().split("/")[1]);
 393             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 394         } else {
 395             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", prefixEp);
 396             return null;
 397         }
 398
 399         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType));
 400         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 401         Match match = mb.build();
 402         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "L3prefix", match);
 403         FlowBuilder flowb = base().setId(flowid)
 404             .setPriority(Integer.valueOf(BASE_L3_PRIORITY+prefixLength))
 405             .setMatch(match)
 406             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 407         return flowb.build();
 408     }
 409
 410     private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) {
 411         MatchBuilder mb = new MatchBuilder()
 412                             .setEthernetMatch(new EthernetMatchBuilder()
 413                             .setEthernetDestination(new EthernetDestinationBuilder().
 414                                                         setAddress(MULTICAST_MAC)
 415                                                         .setMask(MULTICAST_MAC).build())
 416                             .build());
 417         addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId())));
 418
 419         Match match = mb.build();
 420         FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "broadcast", match);
 421         FlowBuilder flowb = base().setPriority(Integer.valueOf(140))
 422             .setId(flowId)
 423             .setMatch(match)
 424             .setInstructions(
 425                     instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false),
 426                             groupAction(Long.valueOf(epOrd.getFdId())))));
 427
 428         return flowb.build();
 429     }
 430
 431     private boolean groupExists(NodeId nodeId, Integer fdId) throws Exception {
 432         // Fetch existing GroupTables
 433         if (ctx.getDataBroker() == null) {
 434             return false;
 435         }
 436
 437         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 438         InstanceIdentifier<Node> niid = createNodePath(nodeId);
 439         Optional<Node> r = t.read(LogicalDatastoreType.CONFIGURATION, niid).get();
 440         if (!r.isPresent())
 441             return false;
 442         FlowCapableNode fcn = r.get().getAugmentation(FlowCapableNode.class);
 443         if (fcn == null)
 444             return false;
 445
 446         if (fcn.getGroup() != null) {
 447             for (Group g : fcn.getGroup()) {
 448                 if (g.getGroupId().getValue().equals(Long.valueOf(fdId))) { // Group
 449                                                                             // Exists.
 450                     return true;
 451                 }
 452             }
 453         }
 454         return false;
 455     }
 456
 457     private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) {
 458
 459         if (ctx.getDataBroker() == null) {
 460             return null;
 461         }
 462
 463         MacAddress defaultMacAddress = ROUTER_MAC;
 464
 465         EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId());
 466         InstanceIdentifier<EndpointL3> endpointsIid = InstanceIdentifier.builder(Endpoints.class)
 467             .child(EndpointL3.class, l3Key)
 468             .build();
 469         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 470
 471         Optional<EndpointL3> r;
 472         try {
 473             r = t.read(LogicalDatastoreType.OPERATIONAL, endpointsIid).get();
 474             if (!r.isPresent())
 475                 return defaultMacAddress;
 476             EndpointL3 epL3 = r.get();
 477             if (epL3.getMacAddress() == null) {
 478                 return defaultMacAddress;
 479             } else {
 480                 return epL3.getMacAddress();
 481             }
 482         } catch (Exception e) {
 483             LOG.error("Error reading EndpointL3 {}.{}", l3c, ipAddress, e);
 484             return null;
 485         }
 486     }
 487
 488     private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) {
 489         IndexedTenant indexedTenant = ctx.getPolicyResolver().getTenant(tenantId);
 490         if (indexedTenant == null) {
 491             LOG.debug("Tenant {} is null, cannot get L3 context", tenantId);
 492             return null;
 493         }
 494         L3Context l3c = indexedTenant.resolveL3Context(sn.getId());
 495         return l3c;
 496     }
 497
 498     private Flow createRouterArpFlow(TenantId tenantId, NodeId nodeId, Subnet sn, int l3Id) {
 499         if (sn == null || sn.getVirtualRouterIp() == null) {
 500             LOG.trace("Didn't create routerArpFlow since either subnet or subnet virtual router was null");
 501             return null;
 502         }
 503         /*
 504          * TODO: Li alagalah: This should be new Yang "gateways" list as well,
 505          * that expresses the gateway and prefixes it is interface for. Should
 506          * also check for external.
 507          */
 508         if (sn.getVirtualRouterIp().getIpv4Address() != null) {
 509             String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue();
 510
 511             L3Context l3c = getL3ContextForSubnet(tenantId, sn);
 512             if (l3c == null) {
 513                 LOG.error("No L3 Context found associated with subnet {}", sn.getId());
 514             }
 515
 516             MacAddress routerMac = routerPortMac(l3c, sn.getVirtualRouterIp());
 517             if (routerMac == null) {
 518                 return null;
 519             }
 520
 521             BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue()));
 522
 523             MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match(
 524                     new ArpMatchBuilder().setArpOp(Integer.valueOf(1))
 525                         .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32"))
 526                         .build());
 527             addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id)));
 528
 529             Match match = mb.build();
 530             FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "routerarp", match);
 531             FlowBuilder flowb = base().setPriority(150)
 532                 .setId(flowId)
 533                 .setMatch(match)
 534                 .setInstructions(
 535                         instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac),
 536                                 nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(),
 537                                 nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(),
 538                                 nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue()
 539                                         + ":INPORT")))));
 540             return flowb.build();
 541         } else {
 542             LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId()
 543                 .getValue());
 544             return null;
 545         }
 546
 547     }
 548
 549     private Flow createLocalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) {
 550
 551         // TODO Li alagalah - refactor common code but keep simple method
 552         ArrayList<Instruction> instructions = new ArrayList<>();
 553         List<Action> applyActions = new ArrayList<>();
 554
 555         int order = 0;
 556
 557         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 558         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 559         Action setNextHop;
 560         String nextHop;
 561
 562         // BEGIN L2 LOCAL
 563         nextHop = ofc.getNodeConnectorId().getValue();
 564
 565         long portNum;
 566         try {
 567             portNum = getOfPortNum(ofc.getNodeConnectorId());
 568         } catch (NumberFormatException ex) {
 569             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 570             return null;
 571         }
 572
 573         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 574
 575         // END L2 LOCAL
 576
 577         order += 1;
 578         applyActions.add(setdEPG);
 579         applyActions.add(setdCG);
 580         applyActions.add(setNextHop);
 581         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 582             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 583             .build();
 584         instructions.add(applyActionsIns);
 585
 586         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 587             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 588             .build();
 589         instructions.add(gotoTable);
 590
 591         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 592         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
 593         Match match = mb.build();
 594         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL2", match);
 595         FlowBuilder flowb = base().setId(flowid)
 596             .setPriority(Integer.valueOf(50))
 597             .setMatch(match)
 598             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 599         return flowb.build();
 600     }
 601
 602     private void syncEP(FlowMap flowMap, NodeId nodeId, PolicyInfo policyInfo, Endpoint srcEp, Endpoint destEp)
 603             throws Exception {
 604
 605         // TODO: Conditions messed up, but for now, send policyInfo until this
 606         // is fixed.
 607         EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, destEp);
 608         EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, srcEp);
 609
 610         if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) {
 611             if (destEp.getTenant() == null) {
 612                 LOG.debug("Didn't process endpoint {} due to tenant being null", destEp.getKey());
 613             } else {
 614                 LOG.debug("Didn't process endpoint {} due to EPG(s) being null", destEp.getKey());
 615             }
 616             return;
 617         }
 618         OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class);
 619
 620         if (LocationType.External.equals(ofc.getLocationType())) {
 621             LOG.error("syncEp(): External endpoints should not be seen here.");
 622             return;
 623         }
 624
 625         /*
 626          * Only care about subnets for L3, but fetch them before loop. We need
 627          * the local subnets for setting SRC MAC for routing. All Routing is now
 628          * done locally! YAY! Instead of being shovelled L2 style across network
 629          * ala Helium.
 630          */
 631         List<Subnet> localSubnets = getLocalSubnets(nodeId);
 632         if (localSubnets == null) {
 633             LOG.error("No subnets could be found locally for node: {}", nodeId);
 634             return;
 635         }
 636
 637         if (Objects.equals(ofc.getNodeId(), nodeId)) {
 638             // this is a local endpoint; send to the approppriate local
 639             // port
 640
 641             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 642                 flowMap.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc));
 643             }
 644             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 645             // The Endpoint conflation must end!
 646             if (destEp.getL3Address() == null) {
 647                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 648                 return;
 649             }
 650
 651             for (L3Address l3a : destEp.getL3Address()) {
 652                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 653                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 654                             destEp.getL3Address());
 655                     continue;
 656                 } else {
 657                     for (Subnet localSubnet : localSubnets) {
 658                         Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet);
 659                         if (flow != null) {
 660                             flowMap.writeFlow(nodeId, TABLE_ID, flow);
 661                         } else {
 662                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 663                                     localSubnet.getIpPrefix().getValue());
 664                         }
 665                     }
 666                 }
 667             }
 668         } else {
 669             // this endpoint is on a different switch; send to the
 670             // appropriate tunnel
 671             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 672                 Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc);
 673                 if (remoteL2Flow != null) {
 674                     flowMap.writeFlow(nodeId, TABLE_ID, remoteL2Flow);
 675                 }
 676             } else {
 677                 LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}",
 678                         srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId());
 679             }
 680
 681             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 682             // The Endpoint conflation must end!
 683             if (destEp.getL3Address() == null) {
 684                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 685                 return;
 686             }
 687             for (L3Address l3a : destEp.getL3Address()) {
 688                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 689                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 690                             destEp.getL3Address());
 691                     continue;
 692                 } else {
 693                     for (Subnet localSubnet : localSubnets) {
 694                         Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds,
 695                                 destEpFwdCtxOrds, ofc, localSubnet);
 696                         if (remoteL3Flow != null) {
 697                             flowMap.writeFlow(nodeId, TABLE_ID, remoteL3Flow);
 698                         } else {
 699                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 700                                     localSubnet.getIpPrefix().getValue());
 701                         }
 702                     }
 703                 }
 704             }
 705         } // remote (tunnel)
 706
 707         // }
 708
 709     }
 710
 711     /*
 712      * ################################## DestMapper Flow methods
 713      * ##################################
 714      */
 715     private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
 716             OfOverlayContext ofc, Subnet srcSubnet) {
 717
 718         // TODO Li alagalah - refactor common code but keep simple method
 719
 720         Subnet destSubnet = null;
 721         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 722         if (subnets == null) {
 723             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 724             return null;
 725         }
 726         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 727         for (Subnet subnet : subnets) {
 728             // TODO Li alagalah add IPv6 support
 729             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 730                 destSubnet = subnet;
 731                 break;
 732             }
 733         }
 734         if (destSubnet == null) {
 735             LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 736             return null;
 737         }
 738
 739         if (destSubnet.getVirtualRouterIp() == null) {
 740             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 741                     destL3Address.getKey());
 742             return null;
 743         }
 744
 745         if (srcSubnet.getVirtualRouterIp() == null) {
 746             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 747             return null;
 748         }
 749         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 750         if (destL3c == null || destL3c.getId() == null) {
 751             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 752             return null;
 753         }
 754         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 755         if (srcL3c == null || srcL3c.getId() == null) {
 756             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 757             return null;
 758         }
 759
 760         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 761             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 762                 .getValue(), destL3c.getId().getValue());
 763             return null;
 764         }
 765
 766         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 767         MacAddress epDestMac = destEp.getMacAddress();
 768         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 769
 770         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 771             // This is our final destination, so match on actual EP mac.
 772             matcherMac = epDestMac;
 773         }
 774
 775         ArrayList<Instruction> l3instructions = new ArrayList<>();
 776         List<Action> applyActions = new ArrayList<>();
 777         List<Action> l3ApplyActions = new ArrayList<>();
 778
 779         int order = 0;
 780
 781         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 782         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 783         Action setNextHop;
 784         String nextHop;
 785
 786         // BEGIN L3 LOCAL
 787         nextHop = ofc.getNodeConnectorId().getValue();
 788
 789         long portNum;
 790         try {
 791             portNum = getOfPortNum(ofc.getNodeConnectorId());
 792         } catch (NumberFormatException ex) {
 793             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 794             return null;
 795         }
 796
 797         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 798         // END L3 LOCAL
 799
 800         // Lets not re-write the srcMac if its local.
 801         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
 802             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
 803             l3ApplyActions.add(setDlSrc);
 804         }
 805
 806         Action setDlDst = setDlDstAction(epDestMac);
 807         l3ApplyActions.add(setDlDst);
 808
 809         Action decTtl = decNwTtlAction();
 810         l3ApplyActions.add(decTtl);
 811
 812         order += 1;
 813         applyActions.add(setdEPG);
 814         applyActions.add(setdCG);
 815         applyActions.add(setNextHop);
 816
 817         applyActions.addAll(l3ApplyActions);
 818         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 819             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 820             .build();
 821
 822         l3instructions.add(applyActionsIns);
 823         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 824             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 825             .build();
 826         l3instructions.add(gotoTable);
 827         Layer3Match m = null;
 828         Long etherType = null;
 829         String ikey = null;
 830         if (destL3Address.getIpAddress().getIpv4Address() != null) {
 831             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
 832             etherType = IPv4;
 833             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 834         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
 835             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
 836             etherType = IPv6;
 837             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 838         } else {
 839             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
 840             return null;
 841         }
 842
 843         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
 844             .setLayer3Match(m);
 845         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 846         Match match = mb.build();
 847         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL3", match);
 848         FlowBuilder flowb = base().setId(flowid)
 849             .setPriority(Integer.valueOf(132))
 850             .setMatch(match)
 851             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 852         return flowb.build();
 853     }
 854
 855     private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds,
 856             EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) {
 857
 858         // TODO Li alagalah - refactor common code but keep simple method
 859
 860         // this endpoint is on a different switch; send to the
 861         // appropriate tunnel
 862
 863         ArrayList<Instruction> instructions = new ArrayList<>();
 864         List<Action> applyActions = new ArrayList<>();
 865
 866         int order = 0;
 867
 868         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
 869         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
 870         Action setNextHop;
 871         String nextHop;
 872
 873         // BEGIN TUNNEL HANDLING
 874         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
 875         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
 876         if (tunDst == null) {
 877             LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep);
 878             return null;
 879         }
 880         if (tunPort == null) {
 881             LOG.warn("Failed to get Tunnel Port for NodeId {} with EP {}", nodeId, ep);
 882             return null;
 883         }
 884
 885         Action tundstAction;
 886
 887         if (tunDst.getIpv4Address() != null) {
 888             nextHop = tunDst.getIpv4Address().getValue();
 889             tundstAction = nxLoadTunIPv4Action(nextHop, false);
 890         } else if (tunDst.getIpv6Address() != null) {
 891             // nextHop = tunDst.getIpv6Address().getValue();
 892             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
 893                     ofc.getNodeId());
 894             return null;
 895         } else {
 896             // this shouldn't happen
 897             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
 898             return null;
 899         }
 900
 901         long portNum;
 902         try {
 903             portNum = getOfPortNum(tunPort);
 904         } catch (NumberFormatException ex) {
 905             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 906             return null;
 907         }
 908
 909         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 910         applyActions.add(tundstAction);
 911         // END TUNNEL
 912
 913         order += 1;
 914         applyActions.add(setdEPG);
 915         applyActions.add(setdCG);
 916         applyActions.add(setNextHop);
 917         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 918             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 919             .build();
 920         instructions.add(applyActionsIns);
 921
 922         applyActionsIns = new InstructionBuilder().setOrder(order++)
 923             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 924             .build();
 925
 926         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 927             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 928             .build();
 929         instructions.add(gotoTable);
 930
 931         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 932         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId())));
 933         Match match = mb.build();
 934         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL2", match);
 935         FlowBuilder flowb = base().setId(flowid)
 936             .setPriority(Integer.valueOf(50))
 937             .setMatch(match)
 938             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 939
 940         return flowb.build();
 941     }
 942
 943     private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId,
 944             EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc,
 945             Subnet srcSubnet) {
 946
 947         // TODO Li alagalah - refactor common code but keep simple method
 948
 949         // this endpoint is on a different switch; send to the
 950         // appropriate tunnel
 951         Subnet destSubnet = null;
 952         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 953         if (subnets == null) {
 954             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 955             return null;
 956         }
 957         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 958         for (Subnet subnet : subnets) {
 959             // TODO Li alagalah add IPv6 support
 960             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 961                 destSubnet = subnet;
 962                 break;
 963             }
 964         }
 965         if (destSubnet == null) {
 966             LOG.info("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 967             return null;
 968         }
 969
 970         if (destSubnet.getVirtualRouterIp() == null) {
 971             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 972                     destL3Address.getKey());
 973             return null;
 974         }
 975
 976         if (srcSubnet.getVirtualRouterIp() == null) {
 977             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 978             return null;
 979         }
 980         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 981         if (destL3c == null || destL3c.getId() == null) {
 982             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 983             return null;
 984         }
 985         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 986         if (srcL3c == null || srcL3c.getId() == null) {
 987             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 988             return null;
 989         }
 990
 991         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 992             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 993                 .getValue(), destL3c.getId().getValue());
 994             return null;
 995         }
 996
 997         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 998         MacAddress epDestMac = destEp.getMacAddress();
 999         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
1000         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
1001             // This is our final destination, so match on actual EP mac.
1002             matcherMac = epDestMac;
1003         }
1004         ArrayList<Instruction> l3instructions = new ArrayList<>();
1005         List<Action> applyActions = new ArrayList<>();
1006         List<Action> l3ApplyActions = new ArrayList<>();
1007
1008         int order = 0;
1009
1010         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
1011         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
1012         Action setNextHop;
1013         String nextHop;
1014
1015         // BEGIN TUNNEL HANDLING
1016         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
1017         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
1018         if (tunDst == null) {
1019             LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address);
1020             return null;
1021         }
1022         if (tunPort == null) {
1023             LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, destL3Address);
1024             return null;
1025         }
1026
1027         Action tundstAction;
1028
1029         if (tunDst.getIpv4Address() != null) {
1030             nextHop = tunDst.getIpv4Address().getValue();
1031             tundstAction = nxLoadTunIPv4Action(nextHop, false);
1032         } else if (tunDst.getIpv6Address() != null) {
1033             // nextHop = tunDst.getIpv6Address().getValue();
1034             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
1035                     ofc.getNodeId());
1036             return null;
1037         } else {
1038             // this shouldn't happen
1039             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
1040             return null;
1041         }
1042
1043         long portNum;
1044         try {
1045             portNum = getOfPortNum(tunPort);
1046         } catch (NumberFormatException ex) {
1047             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
1048             return null;
1049         }
1050
1051         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1052         applyActions.add(tundstAction);
1053         // END TUNNEL
1054
1055         order += 1;
1056         applyActions.add(setdEPG);
1057         applyActions.add(setdCG);
1058         applyActions.add(setNextHop);
1059
1060         // Lets not re-write the srcMac if its local.
1061         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
1062             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
1063             l3ApplyActions.add(setDlSrc);
1064         }
1065
1066         Action setDlDst = setDlDstAction(epDestMac);
1067         l3ApplyActions.add(setDlDst);
1068
1069         Action decTtl = decNwTtlAction();
1070         l3ApplyActions.add(decTtl);
1071
1072         applyActions.addAll(l3ApplyActions);
1073         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1074             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1075             .build();
1076
1077         l3instructions.add(applyActionsIns);
1078         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1079             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1080             .build();
1081         l3instructions.add(gotoTable);
1082         Layer3Match m = null;
1083         Long etherType = null;
1084         String ikey = null;
1085         if (destL3Address.getIpAddress().getIpv4Address() != null) {
1086             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
1087             etherType = IPv4;
1088             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
1089         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
1090             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
1091             etherType = IPv6;
1092             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
1093         } else {
1094             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
1095             return null;
1096         }
1097
1098         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
1099             .setLayer3Match(m);
1100         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id())));
1101         Match match = mb.build();
1102         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL3", match);
1103         FlowBuilder flowb = base().setId(flowid)
1104             .setPriority(Integer.valueOf(132))
1105             .setMatch(match)
1106             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
1107         return flowb.build();
1108     }
1109
1110     private NetworkDomainId getEPNetworkContainment(Endpoint endpoint) {
1111         if (endpoint.getNetworkContainment() != null) {
1112             return endpoint.getNetworkContainment();
1113         } else {
1114             /*
1115              * TODO: Be alagalah: Endpoint Refactor: This should be set on input
1116              * which we can't do because of the backwards way endpoints were
1117              * "architected".
1118              */
1119             return ctx.getPolicyResolver()
1120                 .getTenant(endpoint.getTenant())
1121                 .getEndpointGroup(endpoint.getEndpointGroup())
1122                 .getNetworkDomain();
1123         }
1124     }
1125
1126     private HashSet<Subnet> getSubnets(final TenantId tenantId) {
1127
1128         // if (subnetsByTenant.get(tenantId) != null) {
1129         // return subnetsByTenant.get(tenantId);
1130         // }
1131
1132         if (ctx.getDataBroker() == null) {
1133             return null;
1134         }
1135
1136         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
1137         InstanceIdentifier<Tenant> tiid = TenantUtils.tenantIid(tenantId);
1138         Optional<Tenant> tenantInfo;
1139         try {
1140             tenantInfo = t.read(LogicalDatastoreType.CONFIGURATION, tiid).get();
1141         } catch (Exception e) {
1142             LOG.error("Could not read Tenant {}", tenantId, e);
1143             return null;
1144         }
1145
1146         HashSet<Subnet> subnets = new HashSet<Subnet>();
1147
1148         if (!tenantInfo.isPresent()) {
1149             LOG.warn("Tenant {} not found", tenantId);
1150             return null;
1151         }
1152
1153         subnets.addAll(tenantInfo.get().getSubnet());
1154         // subnetsByTenant.put(tenantId, subnets);
1155         return subnets;
1156     }
1157
1158     // Need a method to get subnets for EPs attached to the node locally
1159     // to set the source Mac address for the router interface.
1160     private List<Subnet> getLocalSubnets(NodeId nodeId) {
1161         Collection<Endpoint> endpointsForNode = ctx.getEndpointManager().getEndpointsForNode(nodeId);
1162
1163         List<Subnet> localSubnets = new ArrayList<Subnet>();
1164
1165         for (Endpoint endpoint : endpointsForNode) {
1166             HashSet<Subnet> subnets = getSubnets(endpoint.getTenant());
1167             if (subnets == null) {
1168                 LOG.debug("No local subnets in tenant {} for EP {}.", endpoint.getTenant(), endpoint.getKey());
1169                 continue;
1170             }
1171             NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint);
1172             for (Subnet subnet : subnets) {
1173                 if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) {
1174                     localSubnets.add(subnet);
1175                 }
1176             }
1177         }
1178         return localSubnets;
1179     }
1180
1181     static byte[] bytesFromHexString(String values) {
1182         String target = "";
1183         if (values != null) {
1184             target = values;
1185         }
1186         String[] octets = target.split(":");
1187
1188         byte[] ret = new byte[octets.length];
1189         for (int i = 0; i < octets.length; i++) {
1190             ret[i] = Integer.valueOf(octets[i], 16).byteValue();
1191         }
1192         return ret;
1193     }
1194 }