renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java

   1 /*
   2  * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow;
  10
  11 import static com.google.common.base.Preconditions.checkNotNull;
  12 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP;
  13 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4;
  14 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6;
  15 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch;
  16 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns;
  17 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.createNodePath;
  18 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction;
  19 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch;
  20 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum;
  21 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.gotoTableIns;
  22 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.groupAction;
  23 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.instructions;
  24 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpOpAction;
  25 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpShaAction;
  26 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction;
  27 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction;
  28 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action;
  29 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction;
  30 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction;
  31 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction;
  32 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction;
  33 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction;
  34 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction;
  35 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction;
  36 import static org.opendaylight.groupbasedpolicy.util.DataStoreHelper.readFromDs;
  37
  38 import java.math.BigInteger;
  39 import java.util.ArrayList;
  40 import java.util.Collection;
  41 import java.util.Collections;
  42 import java.util.HashMap;
  43 import java.util.HashSet;
  44 import java.util.List;
  45 import java.util.Map;
  46 import java.util.Map.Entry;
  47 import java.util.Objects;
  48 import java.util.Set;
  49
  50 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  51 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  52 import org.opendaylight.groupbasedpolicy.endpoint.EpKey;
  53 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext;
  54 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.PolicyManager.FlowMap;
  55 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch;
  56 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals;
  57 import org.opendaylight.groupbasedpolicy.resolver.EgKey;
  58 import org.opendaylight.groupbasedpolicy.resolver.PolicyInfo;
  59 import org.opendaylight.groupbasedpolicy.resolver.TenantUtils;
  60 import org.opendaylight.groupbasedpolicy.util.IidFactory;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
  63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.l3.prefix.fields.EndpointL3Gateways;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Prefix;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.EndpointLocation.LocationType;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.L3Context;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.Subnet;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.ethernet.match.fields.EthernetDestinationBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatchBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.Layer3Match;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 106 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 107 import org.slf4j.Logger;
 108 import org.slf4j.LoggerFactory;
 109
 110 import com.google.common.base.Optional;
 111 import com.google.common.base.Strings;
 112 import com.google.common.collect.HashMultimap;
 113 import com.google.common.collect.SetMultimap;
 114 import com.google.common.collect.Sets;
 115
 116 /**
 117  * Manage the table that maps the destination address to the next hop for the
 118  * path as well as applies any relevant routing transformations.
 119  */
 120 public class DestinationMapper extends FlowTable {
 121
 122     protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class);
 123
 124     // TODO Li alagalah: Improve UT coverage for this class.
 125
 126     // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows
 127     // This ensures we have the appropriate network-containment'
 128
 129     public static short TABLE_ID;
 130     /**
 131      * This is the MAC address of the magical router in the sky
 132      */
 133     public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5");
 134     public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00");
 135     public static final Integer BASE_L3_PRIORITY = 100;
 136
 137     public DestinationMapper(OfContext ctx, short tableId) {
 138         super(ctx);
 139         this.TABLE_ID = tableId;
 140     }
 141
 142     Map<TenantId, HashSet<Subnet>> subnetsByTenant = new HashMap<TenantId, HashSet<Subnet>>();
 143
 144     @Override
 145     public short getTableId() {
 146         return TABLE_ID;
 147     }
 148
 149     @Override
 150     public void sync(NodeId nodeId, PolicyInfo policyInfo, FlowMap flowMap) throws Exception {
 151
 152         TenantId currentTenant = null;
 153
 154         flowMap.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null));
 155
 156         SetMultimap<EpKey, EpKey> visitedEps = HashMultimap.create();
 157         Set<EndpointFwdCtxOrdinals> epOrdSet = new HashSet<>();
 158
 159         for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) {
 160             Set<EndpointGroupId> srcEpgIds = new HashSet<>();
 161             if (srcEp.getEndpointGroup() != null)
 162                 srcEpgIds.add(srcEp.getEndpointGroup());
 163             if (srcEp.getEndpointGroups() != null)
 164                 srcEpgIds.addAll(srcEp.getEndpointGroups());
 165
 166             for (EndpointGroupId epgId : srcEpgIds) {
 167                 EgKey epg = new EgKey(srcEp.getTenant(), epgId);
 168                 Set<EgKey> peers = Sets.union(Collections.singleton(epg), policyInfo.getPeers(epg));
 169                 for (EgKey peer : peers) {
 170                     for (Endpoint peerEp : ctx.getEndpointManager().getEndpointsForGroup(peer)) {
 171                         currentTenant = peerEp.getTenant();
 172                         subnetsByTenant.put(currentTenant, getSubnets(currentTenant));
 173                         EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress());
 174                         EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress());
 175
 176                         if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) {
 177                             continue;
 178                         }
 179                         syncEP(flowMap, nodeId, policyInfo, srcEp, peerEp);
 180                         visitedEps.put(srcEpKey, peerEpKey);
 181
 182                         // Process subnets and flood-domains for epPeer
 183                         EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo,
 184                                 peerEp);
 185                         epOrdSet.add(epOrds);
 186                     }
 187                 }
 188             }
 189         }
 190
 191         for (Entry<TenantId, HashSet<Subnet>> subnetEntry : subnetsByTenant.entrySet()) {
 192             if (subnetEntry.getValue() == null) {
 193                 LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey());
 194                 continue;
 195             }
 196             currentTenant = subnetEntry.getKey();
 197             for (Subnet sn : subnetEntry.getValue()) {
 198                 L3Context l3c = getL3ContextForSubnet(currentTenant, sn);
 199                 Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn,
 200                         OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId()));
 201                 if (arpFlow != null) {
 202                     flowMap.writeFlow(nodeId, TABLE_ID, arpFlow);
 203                 } else {
 204                     LOG.debug(
 205                             "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .",
 206                             sn.getIpPrefix().getValue());
 207                 }
 208             }
 209         }
 210
 211         // Write broadcast flows per flood domain.
 212         for (EndpointFwdCtxOrdinals epOrd : epOrdSet) {
 213             if (groupExists(nodeId, epOrd.getFdId())) {
 214                 flowMap.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd));
 215             }
 216         }
 217
 218         // L3 Prefix Endpoint handling
 219         Collection<EndpointL3Prefix> prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(currentTenant);
 220         if (prefixEps != null) {
 221             LOG.trace("DestinationMapper - Processing L3PrefixEndpoints");
 222             for (EndpointL3Prefix prefixEp : prefixEps) {
 223                 Flow prefixFlow = createL3PrefixFlow(prefixEp, policyInfo, nodeId);
 224                 if (prefixFlow != null) {
 225                     flowMap.writeFlow(nodeId, TABLE_ID, prefixFlow);
 226                     LOG.trace("Wrote L3Prefix flow");
 227                 }
 228             }
 229         }
 230
 231     }
 232
 233     // set up next-hop destinations for all the endpoints in the endpoint
 234     // group on the node
 235
 236     private Flow createL3PrefixFlow(EndpointL3Prefix prefixEp, PolicyInfo policyInfo, NodeId nodeId) throws Exception {
 237         /*
 238          * Priority: 100+lengthprefix
 239          * Match: prefix, l3c, "mac address of router" ?
 240          * Action:
 241          * - set Reg2, Reg3 for L3Ep by L2Ep ?
 242          * - if external,
 243          * - Reg7: use switch location external port else punt for now
 244          * - if internal
 245          * - Reg7: grab L2Ep from L3Ep and use its location info
 246          * - goto_table: POLENF (will check there for external on EP)
 247          */
 248
 249         ReadOnlyTransaction rTx = ctx.getDataBroker().newReadOnlyTransaction();
 250         // TODO Bug #3440 Target: Be - should support for more than first gateway.
 251         EndpointL3Gateways l3Gateway = prefixEp.getEndpointL3Gateways().get(0);
 252         Optional<EndpointL3> optL3Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 253                 IidFactory.l3EndpointIid(l3Gateway.getL3Context(), l3Gateway.getIpAddress()), rTx);
 254         if (!optL3Ep.isPresent()) {
 255             LOG.error("createL3PrefixFlow - L3Endpoint gateway {} for L3Prefix {} not found.", l3Gateway, prefixEp);
 256             return null;
 257         }
 258         EndpointL3 l3Ep = optL3Ep.get();
 259         Optional<Endpoint> optL2Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 260                 IidFactory.endpointIid(l3Ep.getL2Context(), l3Ep.getMacAddress()), rTx);
 261         if (!optL2Ep.isPresent()) {
 262             LOG.error("createL3PrefixFlow - L2Endpoint for L3Gateway {} not found.", l3Ep);
 263             return null;
 264         }
 265         Endpoint l2Ep = optL2Ep.get();
 266         EndpointFwdCtxOrdinals epFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, l2Ep);
 267
 268         NetworkDomainId epNetworkContainment = getEPNetworkContainment(l2Ep);
 269
 270         MacAddress epDestMac = l2Ep.getMacAddress();
 271         MacAddress destSubnetGatewayMac = l2Ep.getMacAddress();
 272
 273         ArrayList<Instruction> l3instructions = new ArrayList<>();
 274         List<Action> applyActions = new ArrayList<>();
 275         List<Action> l3ApplyActions = new ArrayList<>();
 276
 277         int order = 0;
 278
 279         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 280         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 281         Action setNextHop;
 282         String nextHop=null;
 283
 284         OfOverlayContext ofc = l2Ep.getAugmentation(OfOverlayContext.class);
 285         LocationType location;
 286
 287         if (ofc != null && ofc.getLocationType() != null) {
 288             location = ofc.getLocationType();
 289         } else if (ofc != null) {
 290             // Augmentation, but using default location
 291             location = LocationType.Internal;
 292         } else {
 293             LOG.info("createL3PrefixFlow - Endpoint {} had no augmentation.", l2Ep);
 294             return null;
 295         }
 296
 297         long portNum = -1;
 298
 299         if (location.equals(LocationType.Internal)) {
 300             checkNotNull(ofc.getNodeConnectorId());
 301             nextHop = ofc.getNodeConnectorId().getValue();
 302             try {
 303                 portNum = getOfPortNum(ofc.getNodeConnectorId());
 304             } catch (NumberFormatException ex) {
 305                 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 306                 return null;
 307             }
 308
 309         } else {
 310             // External
 311             Set<NodeConnectorId> externalPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
 312             checkNotNull(externalPorts);
 313             for (NodeConnectorId externalPort : externalPorts) {
 314                 // TODO Bug #3440 Target: Be - should support for more than first external port.
 315                 //TODO Bug 3546 - Difficult: External port is unrelated to Tenant, L3C, L2BD..
 316                 nextHop = externalPort.getValue();
 317                 try {
 318                     portNum = getOfPortNum(externalPort);
 319                 } catch (NumberFormatException ex) {
 320                     LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 321                     return null;
 322                 }
 323                 continue;
 324             }
 325         }
 326
 327         if (Strings.isNullOrEmpty(nextHop)
 328                 || portNum == -1) {
 329             LOG.error("createL3Prefix - Cannot find nodeConnectorId for {} for Prefix: ", l2Ep, prefixEp);
 330             return null;
 331         }
 332         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 333
 334         Action setDlDst = setDlDstAction(epDestMac);
 335         l3ApplyActions.add(setDlDst);
 336
 337         Action decTtl = decNwTtlAction();
 338         l3ApplyActions.add(decTtl);
 339
 340         order += 1;
 341         applyActions.add(setdEPG);
 342         applyActions.add(setdCG);
 343         applyActions.add(setNextHop);
 344
 345         applyActions.addAll(l3ApplyActions);
 346         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 347             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 348             .build();
 349
 350         l3instructions.add(applyActionsIns);
 351         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 352             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 353             .build();
 354         l3instructions.add(gotoTable);
 355
 356         Layer3Match m = null;
 357         Long etherType = null;
 358         String ikey = null;
 359         Integer prefixLength=0;
 360         if (prefixEp.getIpPrefix().getIpv4Prefix() != null) {
 361             ikey = prefixEp.getIpPrefix().getIpv4Prefix().getValue();
 362             etherType = IPv4;
 363             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv4Prefix().getValue().split("/")[1]);
 364             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 365         } else if (prefixEp.getIpPrefix().getIpv6Prefix() != null) {
 366             ikey = prefixEp.getIpPrefix().getIpv6Prefix().getValue();
 367             etherType = IPv6;
 368             /*
 369              *  This will result in flows with priority between 100-228, but since its matching on IPv6 prefix as well
 370              *  this shouldn't pose and issue, as the priority is more important within the address space of the matcher,
 371              *  even though technically flows are processed in priority order.
 372              */
 373
 374             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv6Prefix().getValue().split("/")[1]);
 375             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 376         } else {
 377             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", prefixEp);
 378             return null;
 379         }
 380
 381         FlowId flowid = new FlowId(new StringBuilder().append(Integer.toString(epFwdCtxOrds.getL3Id()))
 382             .append("|l3prefix|")
 383             .append(ikey)
 384             .append("|")
 385             .append(destSubnetGatewayMac)
 386             .append("|")
 387             .append(nextHop)
 388             .toString());
 389         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, etherType));
 390 //        MatchBuilder mb = new MatchBuilder();//.setLayer3Match(m);
 391         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 392         FlowBuilder flowb = base().setId(flowid)
 393             .setPriority(Integer.valueOf(BASE_L3_PRIORITY+prefixLength))
 394             .setMatch(mb.build())
 395             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 396         return flowb.build();
 397     }
 398
 399     private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) {
 400         FlowId flowId = new FlowId("broadcast|" + epOrd.getFdId());
 401         MatchBuilder mb = new MatchBuilder().setEthernetMatch(new EthernetMatchBuilder().setEthernetDestination(
 402                 new EthernetDestinationBuilder().setAddress(MULTICAST_MAC).setMask(MULTICAST_MAC).build()).build());
 403         addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId())));
 404
 405         FlowBuilder flowb = base().setPriority(Integer.valueOf(140))
 406             .setId(flowId)
 407             .setMatch(mb.build())
 408             .setInstructions(
 409                     instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false),
 410                             groupAction(Long.valueOf(epOrd.getFdId())))));
 411
 412         return flowb.build();
 413     }
 414
 415     private boolean groupExists(NodeId nodeId, Integer fdId) throws Exception {
 416         // Fetch existing GroupTables
 417         if (ctx.getDataBroker() == null) {
 418             return false;
 419         }
 420
 421         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 422         InstanceIdentifier<Node> niid = createNodePath(nodeId);
 423         Optional<Node> r = t.read(LogicalDatastoreType.CONFIGURATION, niid).get();
 424         if (!r.isPresent())
 425             return false;
 426         FlowCapableNode fcn = r.get().getAugmentation(FlowCapableNode.class);
 427         if (fcn == null)
 428             return false;
 429
 430         if (fcn.getGroup() != null) {
 431             for (Group g : fcn.getGroup()) {
 432                 if (g.getGroupId().getValue().equals(Long.valueOf(fdId))) { // Group
 433                                                                             // Exists.
 434                     return true;
 435                 }
 436             }
 437         }
 438         return false;
 439     }
 440
 441     private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) {
 442
 443         if (ctx.getDataBroker() == null) {
 444             return null;
 445         }
 446
 447         MacAddress defaultMacAddress = ROUTER_MAC;
 448
 449         EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId());
 450         InstanceIdentifier<EndpointL3> endpointsIid = InstanceIdentifier.builder(Endpoints.class)
 451             .child(EndpointL3.class, l3Key)
 452             .build();
 453         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 454
 455         Optional<EndpointL3> r;
 456         try {
 457             r = t.read(LogicalDatastoreType.OPERATIONAL, endpointsIid).get();
 458             if (!r.isPresent())
 459                 return defaultMacAddress;
 460             EndpointL3 epL3 = r.get();
 461             if (epL3.getMacAddress() == null) {
 462                 return defaultMacAddress;
 463             } else {
 464                 return epL3.getMacAddress();
 465             }
 466         } catch (Exception e) {
 467             LOG.error("Error reading EndpointL3 {}.{}", l3c, ipAddress, e);
 468             return null;
 469         }
 470     }
 471
 472     private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) {
 473         L3Context l3c = ctx.getPolicyResolver().getTenant(tenantId).resolveL3Context(sn.getId());
 474         return l3c;
 475     }
 476
 477     private Flow createRouterArpFlow(TenantId tenantId, NodeId nodeId, Subnet sn, int l3Id) {
 478         if (sn == null || sn.getVirtualRouterIp() == null) {
 479             LOG.trace("Didn't create routerArpFlow since either subnet or subnet virtual router was null");
 480             return null;
 481         }
 482         /*
 483          * TODO: Li alagalah: This should be new Yang "gateways" list as well,
 484          * that expresses the gateway and prefixes it is interface for. Should
 485          * also check for external.
 486          */
 487         if (sn.getVirtualRouterIp().getIpv4Address() != null) {
 488             String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue();
 489
 490             L3Context l3c = getL3ContextForSubnet(tenantId, sn);
 491             if (l3c == null) {
 492                 LOG.error("No L3 Context found associated with subnet {}", sn.getId());
 493             }
 494
 495             MacAddress routerMac = routerPortMac(l3c, sn.getVirtualRouterIp());
 496             if (routerMac == null) {
 497                 return null;
 498             }
 499
 500             BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue()));
 501
 502             FlowId flowId = new FlowId(new StringBuffer().append("routerarp|")
 503                 .append(sn.getId().getValue())
 504                 .append("|")
 505                 .append(ikey)
 506                 .append("|")
 507                 .append(l3Id)
 508                 .toString());
 509             MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match(
 510                     new ArpMatchBuilder().setArpOp(Integer.valueOf(1))
 511                         .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32"))
 512                         .build());
 513             addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id)));
 514
 515             FlowBuilder flowb = base().setPriority(150)
 516                 .setId(flowId)
 517                 .setMatch(mb.build())
 518                 .setInstructions(
 519                         instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac),
 520                                 nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(),
 521                                 nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(),
 522                                 nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue()
 523                                         + ":INPORT")))));
 524             return flowb.build();
 525         } else {
 526             LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId()
 527                 .getValue());
 528             return null;
 529         }
 530
 531     }
 532
 533     private Flow createLocalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) {
 534
 535         // TODO Li alagalah - refactor common code but keep simple method
 536         ArrayList<Instruction> instructions = new ArrayList<>();
 537         List<Action> applyActions = new ArrayList<>();
 538
 539         int order = 0;
 540
 541         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 542         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 543         Action setNextHop;
 544         String nextHop;
 545
 546         // BEGIN L2 LOCAL
 547         nextHop = ofc.getNodeConnectorId().getValue();
 548
 549         long portNum;
 550         try {
 551             portNum = getOfPortNum(ofc.getNodeConnectorId());
 552         } catch (NumberFormatException ex) {
 553             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 554             return null;
 555         }
 556
 557         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 558
 559         // END L2 LOCAL
 560
 561         order += 1;
 562         applyActions.add(setdEPG);
 563         applyActions.add(setdCG);
 564         applyActions.add(setNextHop);
 565         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 566             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 567             .build();
 568         instructions.add(applyActionsIns);
 569
 570         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 571             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 572             .build();
 573         instructions.add(gotoTable);
 574
 575         FlowId flowid = new FlowId(new StringBuilder().append(epFwdCtxOrds.getBdId())
 576             .append("|l2|")
 577             .append(ep.getMacAddress().getValue())
 578             .append("|")
 579             .append(nextHop)
 580             .toString());
 581         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 582         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
 583         FlowBuilder flowb = base().setId(flowid)
 584             .setPriority(Integer.valueOf(50))
 585             .setMatch(mb.build())
 586             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 587         return flowb.build();
 588     }
 589
 590     private void syncEP(FlowMap flowMap, NodeId nodeId, PolicyInfo policyInfo, Endpoint srcEp, Endpoint destEp)
 591             throws Exception {
 592
 593         // TODO: Conditions messed up, but for now, send policyInfo until this
 594         // is fixed.
 595         EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, destEp);
 596         EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, srcEp);
 597
 598         if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) {
 599             LOG.trace("Didn't process endpoint due to either tenant, or EPG(s) being null", destEp.getKey());
 600             return;
 601         }
 602         OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class);
 603
 604         // ////////////////////////////////////////////////////////////////////////////////////////
 605         /*
 606          * NOT HANDLING EXTERNALS TODO: alagalah Li: External Gateway
 607          * functionality needed here.
 608          */
 609         if (LocationType.External.equals(ofc.getLocationType())) {
 610             // XXX - TODO - perform NAT and send to the external network
 611             // TODO: Use case neutron gateway interface
 612             LOG.warn("External endpoints not yet supported");
 613             return;
 614         }
 615
 616         /*
 617          * Only care about subnets for L3, but fetch them before loop. We need
 618          * the local subnets for setting SRC MAC for routing. All Routing is now
 619          * done locally! YAY! Instead of being shovelled L2 style across network
 620          * ala Helium.
 621          */
 622         List<Subnet> localSubnets = getLocalSubnets(nodeId);
 623         if (localSubnets == null) {
 624             LOG.error("No subnets could be found locally for node: {}", nodeId);
 625             return;
 626         }
 627
 628         if (Objects.equals(ofc.getNodeId(), nodeId)) {
 629             // this is a local endpoint; send to the approppriate local
 630             // port
 631
 632             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 633                 flowMap.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc));
 634             }
 635             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 636             // The Endpoint conflation must end!
 637             if (destEp.getL3Address() == null) {
 638                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 639                 return;
 640             }
 641
 642             for (L3Address l3a : destEp.getL3Address()) {
 643                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 644                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 645                             destEp.getL3Address());
 646                     continue;
 647                 } else {
 648                     for (Subnet localSubnet : localSubnets) {
 649                         Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet);
 650                         if (flow != null) {
 651                             flowMap.writeFlow(nodeId, TABLE_ID, flow);
 652                         } else {
 653                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 654                                     localSubnet.getIpPrefix().getValue());
 655                         }
 656                     }
 657                 }
 658             }
 659         } else {
 660             // this endpoint is on a different switch; send to the
 661             // appropriate tunnel
 662             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 663                 Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc);
 664                 if (remoteL2Flow != null) {
 665                     flowMap.writeFlow(nodeId, TABLE_ID, remoteL2Flow);
 666                 }
 667             } else {
 668                 LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}",
 669                         srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId());
 670             }
 671
 672             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 673             // The Endpoint conflation must end!
 674             if (destEp.getL3Address() == null) {
 675                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 676                 return;
 677             }
 678             for (L3Address l3a : destEp.getL3Address()) {
 679                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 680                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 681                             destEp.getL3Address());
 682                     continue;
 683                 } else {
 684                     for (Subnet localSubnet : localSubnets) {
 685                         Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds,
 686                                 destEpFwdCtxOrds, ofc, localSubnet);
 687                         if (remoteL3Flow != null) {
 688                             flowMap.writeFlow(nodeId, TABLE_ID, remoteL3Flow);
 689                         } else {
 690                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 691                                     localSubnet.getIpPrefix().getValue());
 692                         }
 693                     }
 694                 }
 695             }
 696         } // remote (tunnel)
 697
 698         // }
 699
 700     }
 701
 702     /*
 703      * ################################## DestMapper Flow methods
 704      * ##################################
 705      */
 706     private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
 707             OfOverlayContext ofc, Subnet srcSubnet) {
 708
 709         // TODO Li alagalah - refactor common code but keep simple method
 710
 711         Subnet destSubnet = null;
 712         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 713         if (subnets == null) {
 714             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 715             return null;
 716         }
 717         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 718         for (Subnet subnet : subnets) {
 719             // TODO Li alagalah add IPv6 support
 720             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 721                 destSubnet = subnet;
 722                 break;
 723             }
 724         }
 725         if (destSubnet == null) {
 726             LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 727             return null;
 728         }
 729
 730         if (destSubnet.getVirtualRouterIp() == null) {
 731             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 732                     destL3Address.getKey());
 733             return null;
 734         }
 735
 736         if (srcSubnet.getVirtualRouterIp() == null) {
 737             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 738             return null;
 739         }
 740         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 741         if (destL3c == null || destL3c.getId() == null) {
 742             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 743             return null;
 744         }
 745         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 746         if (srcL3c == null || srcL3c.getId() == null) {
 747             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 748             return null;
 749         }
 750
 751         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 752             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 753                 .getValue(), destL3c.getId().getValue());
 754             return null;
 755         }
 756
 757         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 758         MacAddress epDestMac = destEp.getMacAddress();
 759         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 760
 761         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 762             // This is our final destination, so match on actual EP mac.
 763             matcherMac = epDestMac;
 764         }
 765
 766         ArrayList<Instruction> l3instructions = new ArrayList<>();
 767         List<Action> applyActions = new ArrayList<>();
 768         List<Action> l3ApplyActions = new ArrayList<>();
 769
 770         int order = 0;
 771
 772         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 773         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 774         Action setNextHop;
 775         String nextHop;
 776
 777         // BEGIN L3 LOCAL
 778         nextHop = ofc.getNodeConnectorId().getValue();
 779
 780         long portNum;
 781         try {
 782             portNum = getOfPortNum(ofc.getNodeConnectorId());
 783         } catch (NumberFormatException ex) {
 784             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 785             return null;
 786         }
 787
 788         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 789         // END L3 LOCAL
 790
 791         // Lets not re-write the srcMac if its local.
 792         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
 793             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
 794             l3ApplyActions.add(setDlSrc);
 795         }
 796
 797         Action setDlDst = setDlDstAction(epDestMac);
 798         l3ApplyActions.add(setDlDst);
 799
 800         Action decTtl = decNwTtlAction();
 801         l3ApplyActions.add(decTtl);
 802
 803         order += 1;
 804         applyActions.add(setdEPG);
 805         applyActions.add(setdCG);
 806         applyActions.add(setNextHop);
 807
 808         applyActions.addAll(l3ApplyActions);
 809         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 810             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 811             .build();
 812
 813         l3instructions.add(applyActionsIns);
 814         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 815             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 816             .build();
 817         l3instructions.add(gotoTable);
 818         Layer3Match m = null;
 819         Long etherType = null;
 820         String ikey = null;
 821         if (destL3Address.getIpAddress().getIpv4Address() != null) {
 822             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
 823             etherType = IPv4;
 824             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 825         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
 826             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
 827             etherType = IPv6;
 828             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 829         } else {
 830             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
 831             return null;
 832         }
 833
 834         FlowId flowid = new FlowId(new StringBuilder().append(Integer.toString(epFwdCtxOrds.getL3Id()))
 835             .append("|l3|")
 836             .append(ikey)
 837             .append("|")
 838             .append(Integer.toString(epFwdCtxOrds.getEpgId()))
 839             .append("|")
 840             .append(Integer.toString(epFwdCtxOrds.getCgId()))
 841             .append("|")
 842             .append(matcherMac)
 843             .append("|")
 844             .append(destSubnetGatewayMac)
 845             .append("|")
 846             .append(nextHop)
 847             .toString());
 848         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
 849             .setLayer3Match(m);
 850         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 851         FlowBuilder flowb = base().setId(flowid)
 852             .setPriority(Integer.valueOf(132))
 853             .setMatch(mb.build())
 854             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 855         return flowb.build();
 856     }
 857
 858     private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds,
 859             EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) {
 860
 861         // TODO Li alagalah - refactor common code but keep simple method
 862
 863         // this endpoint is on a different switch; send to the
 864         // appropriate tunnel
 865
 866         ArrayList<Instruction> instructions = new ArrayList<>();
 867         List<Action> applyActions = new ArrayList<>();
 868
 869         int order = 0;
 870
 871         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
 872         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
 873         Action setNextHop;
 874         String nextHop;
 875
 876         // BEGIN TUNNEL HANDLING
 877         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
 878         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
 879         if (tunDst == null) {
 880             LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep);
 881             return null;
 882         }
 883         if (tunPort == null) {
 884             LOG.warn("Failed to get Tunnel Port for NodeId {} with EP {}", nodeId, ep);
 885             return null;
 886         }
 887
 888         Action tundstAction;
 889
 890         if (tunDst.getIpv4Address() != null) {
 891             nextHop = tunDst.getIpv4Address().getValue();
 892             tundstAction = nxLoadTunIPv4Action(nextHop, false);
 893         } else if (tunDst.getIpv6Address() != null) {
 894             // nextHop = tunDst.getIpv6Address().getValue();
 895             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
 896                     ofc.getNodeId());
 897             return null;
 898         } else {
 899             // this shouldn't happen
 900             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
 901             return null;
 902         }
 903
 904         long portNum;
 905         try {
 906             portNum = getOfPortNum(tunPort);
 907         } catch (NumberFormatException ex) {
 908             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 909             return null;
 910         }
 911
 912         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 913         applyActions.add(tundstAction);
 914         // END TUNNEL
 915
 916         order += 1;
 917         applyActions.add(setdEPG);
 918         applyActions.add(setdCG);
 919         applyActions.add(setNextHop);
 920         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 921             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 922             .build();
 923         instructions.add(applyActionsIns);
 924
 925         applyActionsIns = new InstructionBuilder().setOrder(order++)
 926             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 927             .build();
 928
 929         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 930             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 931             .build();
 932         instructions.add(gotoTable);
 933
 934         FlowId flowid = new FlowId(new StringBuilder().append(destEpFwdCtxOrds.getBdId())
 935             .append("|l2|")
 936             .append(ep.getMacAddress().getValue())
 937             .append("|")
 938             .append(srcEpFwdCtxOrds.getBdId())
 939             .append("|")
 940             .append(nextHop)
 941             .toString());
 942         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 943         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId())));
 944         FlowBuilder flowb = base().setId(flowid)
 945             .setPriority(Integer.valueOf(50))
 946             .setMatch(mb.build())
 947             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 948
 949         return flowb.build();
 950     }
 951
 952     private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId,
 953             EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc,
 954             Subnet srcSubnet) {
 955
 956         // TODO Li alagalah - refactor common code but keep simple method
 957
 958         // this endpoint is on a different switch; send to the
 959         // appropriate tunnel
 960         Subnet destSubnet = null;
 961         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 962         if (subnets == null) {
 963             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 964             return null;
 965         }
 966         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 967         for (Subnet subnet : subnets) {
 968             // TODO Li alagalah add IPv6 support
 969             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 970                 destSubnet = subnet;
 971                 break;
 972             }
 973         }
 974         if (destSubnet == null) {
 975             LOG.info("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 976             return null;
 977         }
 978
 979         if (destSubnet.getVirtualRouterIp() == null) {
 980             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 981                     destL3Address.getKey());
 982             return null;
 983         }
 984
 985         if (srcSubnet.getVirtualRouterIp() == null) {
 986             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 987             return null;
 988         }
 989         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 990         if (destL3c == null || destL3c.getId() == null) {
 991             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 992             return null;
 993         }
 994         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 995         if (srcL3c == null || srcL3c.getId() == null) {
 996             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 997             return null;
 998         }
 999
1000         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
1001             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
1002                 .getValue(), destL3c.getId().getValue());
1003             return null;
1004         }
1005
1006         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
1007         MacAddress epDestMac = destEp.getMacAddress();
1008         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
1009
1010         ArrayList<Instruction> l3instructions = new ArrayList<>();
1011         List<Action> applyActions = new ArrayList<>();
1012         List<Action> l3ApplyActions = new ArrayList<>();
1013
1014         int order = 0;
1015
1016         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
1017         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
1018         Action setNextHop;
1019         String nextHop;
1020
1021         // BEGIN TUNNEL HANDLING
1022         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
1023         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
1024         if (tunDst == null) {
1025             LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address);
1026             return null;
1027         }
1028         if (tunPort == null) {
1029             LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, destL3Address);
1030             return null;
1031         }
1032
1033         Action tundstAction;
1034
1035         if (tunDst.getIpv4Address() != null) {
1036             nextHop = tunDst.getIpv4Address().getValue();
1037             tundstAction = nxLoadTunIPv4Action(nextHop, false);
1038         } else if (tunDst.getIpv6Address() != null) {
1039             // nextHop = tunDst.getIpv6Address().getValue();
1040             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
1041                     ofc.getNodeId());
1042             return null;
1043         } else {
1044             // this shouldn't happen
1045             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
1046             return null;
1047         }
1048
1049         long portNum;
1050         try {
1051             portNum = getOfPortNum(tunPort);
1052         } catch (NumberFormatException ex) {
1053             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
1054             return null;
1055         }
1056
1057         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1058         applyActions.add(tundstAction);
1059         // END TUNNEL
1060
1061         order += 1;
1062         applyActions.add(setdEPG);
1063         applyActions.add(setdCG);
1064         applyActions.add(setNextHop);
1065
1066         Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
1067         l3ApplyActions.add(setDlSrc);
1068
1069         Action setDlDst = setDlDstAction(epDestMac);
1070         l3ApplyActions.add(setDlDst);
1071
1072         Action decTtl = decNwTtlAction();
1073         l3ApplyActions.add(decTtl);
1074
1075         applyActions.addAll(l3ApplyActions);
1076         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1077             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1078             .build();
1079
1080         l3instructions.add(applyActionsIns);
1081         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1082             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1083             .build();
1084         l3instructions.add(gotoTable);
1085         Layer3Match m = null;
1086         Long etherType = null;
1087         String ikey = null;
1088         if (destL3Address.getIpAddress().getIpv4Address() != null) {
1089             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
1090             etherType = IPv4;
1091             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
1092         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
1093             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
1094             etherType = IPv6;
1095             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
1096         } else {
1097             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
1098             return null;
1099         }
1100
1101         FlowId flowid = new FlowId(new StringBuilder().append(Integer.toString(destEpFwdCtxOrds.getL3Id()))
1102             .append("|l3|")
1103             .append(ikey)
1104             .append("|")
1105             .append(matcherMac)
1106             .append("|")
1107             .append(destSubnetGatewayMac)
1108             .append("|")
1109             .append(srcEpFwdCtxOrds.getL3Id())
1110             .append("|")
1111             .append(nextHop)
1112             .toString());
1113         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
1114             .setLayer3Match(m);
1115         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id())));
1116
1117         FlowBuilder flowb = base().setId(flowid)
1118             .setPriority(Integer.valueOf(132))
1119             .setMatch(mb.build())
1120             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
1121         return flowb.build();
1122     }
1123
1124     private NetworkDomainId getEPNetworkContainment(Endpoint endpoint) {
1125         if (endpoint.getNetworkContainment() != null) {
1126             return endpoint.getNetworkContainment();
1127         } else {
1128             /*
1129              * TODO: Be alagalah: Endpoint Refactor: This should be set on input
1130              * which we can't do because of the backwards way endpoints were
1131              * "architected".
1132              */
1133             return ctx.getPolicyResolver()
1134                 .getTenant(endpoint.getTenant())
1135                 .getEndpointGroup(endpoint.getEndpointGroup())
1136                 .getNetworkDomain();
1137         }
1138     }
1139
1140     private HashSet<Subnet> getSubnets(final TenantId tenantId) {
1141
1142         // if (subnetsByTenant.get(tenantId) != null) {
1143         // return subnetsByTenant.get(tenantId);
1144         // }
1145
1146         if (ctx.getDataBroker() == null) {
1147             return null;
1148         }
1149
1150         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
1151         InstanceIdentifier<Tenant> tiid = TenantUtils.tenantIid(tenantId);
1152         Optional<Tenant> tenantInfo;
1153         try {
1154             tenantInfo = t.read(LogicalDatastoreType.CONFIGURATION, tiid).get();
1155         } catch (Exception e) {
1156             LOG.error("Could not read Tenant {}", tenantId, e);
1157             return null;
1158         }
1159
1160         HashSet<Subnet> subnets = new HashSet<Subnet>();
1161
1162         if (!tenantInfo.isPresent()) {
1163             LOG.warn("Tenant {} not found", tenantId);
1164             return null;
1165         }
1166
1167         subnets.addAll(tenantInfo.get().getSubnet());
1168         // subnetsByTenant.put(tenantId, subnets);
1169         return subnets;
1170     }
1171
1172     // Need a method to get subnets for EPs attached to the node locally
1173     // to set the source Mac address for the router interface.
1174     private List<Subnet> getLocalSubnets(NodeId nodeId) {
1175         Collection<Endpoint> endpointsForNode = ctx.getEndpointManager().getEndpointsForNode(nodeId);
1176
1177         List<Subnet> localSubnets = new ArrayList<Subnet>();
1178
1179         for (Endpoint endpoint : endpointsForNode) {
1180             HashSet<Subnet> subnets = getSubnets(endpoint.getTenant());
1181             if (subnets == null) {
1182                 LOG.error("No local subnets.");
1183                 return null;
1184             }
1185             NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint);
1186             for (Subnet subnet : subnets) {
1187                 if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) {
1188                     localSubnets.add(subnet);
1189                 }
1190             }
1191         }
1192         return localSubnets;
1193     }
1194
1195     static byte[] bytesFromHexString(String values) {
1196         String target = "";
1197         if (values != null) {
1198             target = values;
1199         }
1200         String[] octets = target.split(":");
1201
1202         byte[] ret = new byte[octets.length];
1203         for (int i = 0; i < octets.length; i++) {
1204             ret[i] = Integer.valueOf(octets[i], 16).byteValue();
1205         }
1206         return ret;
1207     }
1208 }