renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java

   1 /*
   2  * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow;
  10
  11 import com.google.common.base.Optional;
  12 import com.google.common.base.Strings;
  13 import com.google.common.collect.HashMultimap;
  14 import com.google.common.collect.SetMultimap;
  15 import com.google.common.collect.Sets;
  16 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  17 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  18 import org.opendaylight.groupbasedpolicy.endpoint.EpKey;
  19 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext;
  20 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.PolicyManager.FlowMap;
  21 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch;
  22 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals;
  23 import org.opendaylight.groupbasedpolicy.resolver.EgKey;
  24 import org.opendaylight.groupbasedpolicy.resolver.PolicyInfo;
  25 import org.opendaylight.groupbasedpolicy.resolver.TenantUtils;
  26 import org.opendaylight.groupbasedpolicy.util.IidFactory;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
  29 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix;
  30 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.l3.prefix.fields.EndpointL3Gateways;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Prefix;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.EndpointLocation.LocationType;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.L3Context;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.Subnet;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.ethernet.match.fields.EthernetDestinationBuilder;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatchBuilder;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.Layer3Match;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
  73 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  74 import org.slf4j.Logger;
  75 import org.slf4j.LoggerFactory;
  76
  77 import java.math.BigInteger;
  78 import java.util.ArrayList;
  79 import java.util.Collection;
  80 import java.util.Collections;
  81 import java.util.HashMap;
  82 import java.util.HashSet;
  83 import java.util.List;
  84 import java.util.Map;
  85 import java.util.Map.Entry;
  86 import java.util.Objects;
  87 import java.util.Set;
  88
  89 import static com.google.common.base.Preconditions.checkNotNull;
  90 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP;
  91 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4;
  92 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6;
  93 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch;
  94 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns;
  95 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.createNodePath;
  96 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction;
  97 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch;
  98 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum;
  99 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.gotoTableIns;
 100 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.groupAction;
 101 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.instructions;
 102 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpOpAction;
 103 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpShaAction;
 104 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction;
 105 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction;
 106 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action;
 107 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction;
 108 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction;
 109 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction;
 110 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction;
 111 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction;
 112 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction;
 113 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction;
 114 import static org.opendaylight.groupbasedpolicy.util.DataStoreHelper.readFromDs;
 115
 116 /**
 117  * Manage the table that maps the destination address to the next hop for the
 118  * path as well as applies any relevant routing transformations.
 119  */
 120 public class DestinationMapper extends FlowTable {
 121
 122     protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class);
 123
 124     // TODO Li alagalah: Improve UT coverage for this class.
 125
 126     // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows
 127     // This ensures we have the appropriate network-containment'
 128
 129     public static short TABLE_ID;
 130     /**
 131      * This is the MAC address of the magical router in the sky
 132      */
 133     public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5");
 134     public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00");
 135     public static final Integer BASE_L3_PRIORITY = 100;
 136
 137     public DestinationMapper(OfContext ctx, short tableId) {
 138         super(ctx);
 139         this.TABLE_ID = tableId;
 140     }
 141
 142     Map<TenantId, HashSet<Subnet>> subnetsByTenant = new HashMap<TenantId, HashSet<Subnet>>();
 143
 144     @Override
 145     public short getTableId() {
 146         return TABLE_ID;
 147     }
 148
 149     @Override
 150     public void sync(NodeId nodeId, PolicyInfo policyInfo, FlowMap flowMap) throws Exception {
 151
 152         TenantId currentTenant = null;
 153
 154         flowMap.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null, TABLE_ID));
 155
 156         SetMultimap<EpKey, EpKey> visitedEps = HashMultimap.create();
 157         Set<EndpointFwdCtxOrdinals> epOrdSet = new HashSet<>();
 158
 159         for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) {
 160             Set<EndpointGroupId> srcEpgIds = new HashSet<>();
 161             if (srcEp.getEndpointGroup() != null)
 162                 srcEpgIds.add(srcEp.getEndpointGroup());
 163             if (srcEp.getEndpointGroups() != null)
 164                 srcEpgIds.addAll(srcEp.getEndpointGroups());
 165
 166             for (EndpointGroupId epgId : srcEpgIds) {
 167                 EgKey epg = new EgKey(srcEp.getTenant(), epgId);
 168                 Set<EgKey> peers = Sets.union(Collections.singleton(epg), policyInfo.getPeers(epg));
 169                 for (EgKey peer : peers) {
 170                     for (Endpoint peerEp : ctx.getEndpointManager().getEndpointsForGroup(peer)) {
 171                         currentTenant = peerEp.getTenant();
 172                         subnetsByTenant.put(currentTenant, getSubnets(currentTenant));
 173                         EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress());
 174                         EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress());
 175
 176                         if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) {
 177                             continue;
 178                         }
 179                         syncEP(flowMap, nodeId, policyInfo, srcEp, peerEp);
 180                         visitedEps.put(srcEpKey, peerEpKey);
 181
 182                         // Process subnets and flood-domains for epPeer
 183                         EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo,
 184                                 peerEp);
 185                         epOrdSet.add(epOrds);
 186                     }
 187                 }
 188             }
 189         }
 190
 191         for (Entry<TenantId, HashSet<Subnet>> subnetEntry : subnetsByTenant.entrySet()) {
 192             if (subnetEntry.getValue() == null) {
 193                 LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey());
 194                 continue;
 195             }
 196             currentTenant = subnetEntry.getKey();
 197             for (Subnet sn : subnetEntry.getValue()) {
 198                 L3Context l3c = getL3ContextForSubnet(currentTenant, sn);
 199                 Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn,
 200                         OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId()));
 201                 if (arpFlow != null) {
 202                     flowMap.writeFlow(nodeId, TABLE_ID, arpFlow);
 203                 } else {
 204                     LOG.debug(
 205                             "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .",
 206                             sn.getIpPrefix().getValue());
 207                 }
 208             }
 209         }
 210
 211         // Write broadcast flows per flood domain.
 212         for (EndpointFwdCtxOrdinals epOrd : epOrdSet) {
 213             if (groupExists(nodeId, epOrd.getFdId())) {
 214                 flowMap.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd));
 215             }
 216         }
 217
 218         // L3 Prefix Endpoint handling
 219         Collection<EndpointL3Prefix> prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(currentTenant);
 220         if (prefixEps != null) {
 221             LOG.trace("DestinationMapper - Processing L3PrefixEndpoints");
 222             for (EndpointL3Prefix prefixEp : prefixEps) {
 223                 Flow prefixFlow = createL3PrefixFlow(prefixEp, policyInfo, nodeId);
 224                 if (prefixFlow != null) {
 225                     flowMap.writeFlow(nodeId, TABLE_ID, prefixFlow);
 226                     LOG.trace("Wrote L3Prefix flow");
 227                 }
 228             }
 229         }
 230
 231     }
 232
 233     // set up next-hop destinations for all the endpoints in the endpoint
 234     // group on the node
 235
 236     private Flow createL3PrefixFlow(EndpointL3Prefix prefixEp, PolicyInfo policyInfo, NodeId nodeId) throws Exception {
 237         /*
 238          * Priority: 100+lengthprefix
 239          * Match: prefix, l3c, "mac address of router" ?
 240          * Action:
 241          * - set Reg2, Reg3 for L3Ep by L2Ep ?
 242          * - if external,
 243          * - Reg7: use switch location external port else punt for now
 244          * - if internal
 245          * - Reg7: grab L2Ep from L3Ep and use its location info
 246          * - goto_table: POLENF (will check there for external on EP)
 247          */
 248
 249         ReadOnlyTransaction rTx = ctx.getDataBroker().newReadOnlyTransaction();
 250         // TODO Bug #3440 Target: Be - should support for more than first gateway.
 251         EndpointL3Gateways l3Gateway = prefixEp.getEndpointL3Gateways().get(0);
 252         Optional<EndpointL3> optL3Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 253                 IidFactory.l3EndpointIid(l3Gateway.getL3Context(), l3Gateway.getIpAddress()), rTx);
 254         if (!optL3Ep.isPresent()) {
 255             LOG.error("createL3PrefixFlow - L3Endpoint gateway {} for L3Prefix {} not found.", l3Gateway, prefixEp);
 256             return null;
 257         }
 258         EndpointL3 l3Ep = optL3Ep.get();
 259         if (l3Ep.getL2Context() == null || l3Ep.getMacAddress() == null) {
 260             LOG.debug("L3 endpoint representing L3 gateway does not contain L2-context or MAC address. {}", l3Ep);
 261             return null;
 262         }
 263         Optional<Endpoint> optL2Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 264                 IidFactory.endpointIid(l3Ep.getL2Context(), l3Ep.getMacAddress()), rTx);
 265         if (!optL2Ep.isPresent()) {
 266             LOG.error("createL3PrefixFlow - L2Endpoint for L3Gateway {} not found.", l3Ep);
 267             return null;
 268         }
 269         Endpoint l2Ep = optL2Ep.get();
 270         EndpointFwdCtxOrdinals epFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, l2Ep);
 271
 272         NetworkDomainId epNetworkContainment = getEPNetworkContainment(l2Ep);
 273
 274         MacAddress epDestMac = l2Ep.getMacAddress();
 275         MacAddress destSubnetGatewayMac = l2Ep.getMacAddress();
 276
 277         ArrayList<Instruction> l3instructions = new ArrayList<>();
 278         List<Action> applyActions = new ArrayList<>();
 279         List<Action> l3ApplyActions = new ArrayList<>();
 280
 281         int order = 0;
 282
 283         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 284         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 285         Action setNextHop;
 286         String nextHop=null;
 287
 288         OfOverlayContext ofc = l2Ep.getAugmentation(OfOverlayContext.class);
 289         LocationType location;
 290
 291         if (ofc != null && ofc.getLocationType() != null) {
 292             location = ofc.getLocationType();
 293         } else if (ofc != null) {
 294             // Augmentation, but using default location
 295             location = LocationType.Internal;
 296         } else {
 297             LOG.info("createL3PrefixFlow - Endpoint {} had no augmentation.", l2Ep);
 298             return null;
 299         }
 300
 301         long portNum = -1;
 302
 303         if (location.equals(LocationType.Internal)) {
 304             checkNotNull(ofc.getNodeConnectorId());
 305             nextHop = ofc.getNodeConnectorId().getValue();
 306             try {
 307                 portNum = getOfPortNum(ofc.getNodeConnectorId());
 308             } catch (NumberFormatException ex) {
 309                 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 310                 return null;
 311             }
 312
 313         } else {
 314             // External
 315             Set<NodeConnectorId> externalPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
 316             checkNotNull(externalPorts);
 317             for (NodeConnectorId externalPort : externalPorts) {
 318                 // TODO Bug #3440 Target: Be - should support for more than first external port.
 319                 //TODO Bug 3546 - Difficult: External port is unrelated to Tenant, L3C, L2BD..
 320                 nextHop = externalPort.getValue();
 321                 try {
 322                     portNum = getOfPortNum(externalPort);
 323                 } catch (NumberFormatException ex) {
 324                     LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 325                     return null;
 326                 }
 327                 continue;
 328             }
 329         }
 330
 331         if (Strings.isNullOrEmpty(nextHop)
 332                 || portNum == -1) {
 333             LOG.error("createL3Prefix - Cannot find nodeConnectorId for {} for Prefix: ", l2Ep, prefixEp);
 334             return null;
 335         }
 336         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 337
 338         Action setDlDst = setDlDstAction(epDestMac);
 339         l3ApplyActions.add(setDlDst);
 340
 341         Action decTtl = decNwTtlAction();
 342         l3ApplyActions.add(decTtl);
 343
 344         order += 1;
 345         applyActions.add(setdEPG);
 346         applyActions.add(setdCG);
 347         applyActions.add(setNextHop);
 348
 349         applyActions.addAll(l3ApplyActions);
 350         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 351             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 352             .build();
 353
 354         l3instructions.add(applyActionsIns);
 355         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 356             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 357             .build();
 358         l3instructions.add(gotoTable);
 359
 360         Layer3Match m = null;
 361         Long etherType = null;
 362         String ikey = null;
 363         Integer prefixLength=0;
 364         if (prefixEp.getIpPrefix().getIpv4Prefix() != null) {
 365             ikey = prefixEp.getIpPrefix().getIpv4Prefix().getValue();
 366             etherType = IPv4;
 367             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv4Prefix().getValue().split("/")[1]);
 368             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 369         } else if (prefixEp.getIpPrefix().getIpv6Prefix() != null) {
 370             ikey = prefixEp.getIpPrefix().getIpv6Prefix().getValue();
 371             etherType = IPv6;
 372             /*
 373              *  This will result in flows with priority between 100-228, but since its matching on IPv6 prefix as well
 374              *  this shouldn't pose and issue, as the priority is more important within the address space of the matcher,
 375              *  even though technically flows are processed in priority order.
 376              */
 377
 378             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv6Prefix().getValue().split("/")[1]);
 379             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 380         } else {
 381             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", prefixEp);
 382             return null;
 383         }
 384
 385         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, etherType));
 386         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 387         Match match = mb.build();
 388         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "L3prefix", match);
 389         FlowBuilder flowb = base().setId(flowid)
 390             .setPriority(Integer.valueOf(BASE_L3_PRIORITY+prefixLength))
 391             .setMatch(match)
 392             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 393         return flowb.build();
 394     }
 395
 396     private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) {
 397         MatchBuilder mb = new MatchBuilder()
 398                             .setEthernetMatch(new EthernetMatchBuilder()
 399                             .setEthernetDestination(new EthernetDestinationBuilder().
 400                                                         setAddress(MULTICAST_MAC)
 401                                                         .setMask(MULTICAST_MAC).build())
 402                             .build());
 403         addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId())));
 404
 405         Match match = mb.build();
 406         FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "broadcast", match);
 407         FlowBuilder flowb = base().setPriority(Integer.valueOf(140))
 408             .setId(flowId)
 409             .setMatch(match)
 410             .setInstructions(
 411                     instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false),
 412                             groupAction(Long.valueOf(epOrd.getFdId())))));
 413
 414         return flowb.build();
 415     }
 416
 417     private boolean groupExists(NodeId nodeId, Integer fdId) throws Exception {
 418         // Fetch existing GroupTables
 419         if (ctx.getDataBroker() == null) {
 420             return false;
 421         }
 422
 423         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 424         InstanceIdentifier<Node> niid = createNodePath(nodeId);
 425         Optional<Node> r = t.read(LogicalDatastoreType.CONFIGURATION, niid).get();
 426         if (!r.isPresent())
 427             return false;
 428         FlowCapableNode fcn = r.get().getAugmentation(FlowCapableNode.class);
 429         if (fcn == null)
 430             return false;
 431
 432         if (fcn.getGroup() != null) {
 433             for (Group g : fcn.getGroup()) {
 434                 if (g.getGroupId().getValue().equals(Long.valueOf(fdId))) { // Group
 435                                                                             // Exists.
 436                     return true;
 437                 }
 438             }
 439         }
 440         return false;
 441     }
 442
 443     private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) {
 444
 445         if (ctx.getDataBroker() == null) {
 446             return null;
 447         }
 448
 449         MacAddress defaultMacAddress = ROUTER_MAC;
 450
 451         EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId());
 452         InstanceIdentifier<EndpointL3> endpointsIid = InstanceIdentifier.builder(Endpoints.class)
 453             .child(EndpointL3.class, l3Key)
 454             .build();
 455         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 456
 457         Optional<EndpointL3> r;
 458         try {
 459             r = t.read(LogicalDatastoreType.OPERATIONAL, endpointsIid).get();
 460             if (!r.isPresent())
 461                 return defaultMacAddress;
 462             EndpointL3 epL3 = r.get();
 463             if (epL3.getMacAddress() == null) {
 464                 return defaultMacAddress;
 465             } else {
 466                 return epL3.getMacAddress();
 467             }
 468         } catch (Exception e) {
 469             LOG.error("Error reading EndpointL3 {}.{}", l3c, ipAddress, e);
 470             return null;
 471         }
 472     }
 473
 474     private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) {
 475         L3Context l3c = ctx.getPolicyResolver().getTenant(tenantId).resolveL3Context(sn.getId());
 476         return l3c;
 477     }
 478
 479     private Flow createRouterArpFlow(TenantId tenantId, NodeId nodeId, Subnet sn, int l3Id) {
 480         if (sn == null || sn.getVirtualRouterIp() == null) {
 481             LOG.trace("Didn't create routerArpFlow since either subnet or subnet virtual router was null");
 482             return null;
 483         }
 484         /*
 485          * TODO: Li alagalah: This should be new Yang "gateways" list as well,
 486          * that expresses the gateway and prefixes it is interface for. Should
 487          * also check for external.
 488          */
 489         if (sn.getVirtualRouterIp().getIpv4Address() != null) {
 490             String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue();
 491
 492             L3Context l3c = getL3ContextForSubnet(tenantId, sn);
 493             if (l3c == null) {
 494                 LOG.error("No L3 Context found associated with subnet {}", sn.getId());
 495             }
 496
 497             MacAddress routerMac = routerPortMac(l3c, sn.getVirtualRouterIp());
 498             if (routerMac == null) {
 499                 return null;
 500             }
 501
 502             BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue()));
 503
 504             MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match(
 505                     new ArpMatchBuilder().setArpOp(Integer.valueOf(1))
 506                         .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32"))
 507                         .build());
 508             addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id)));
 509
 510             Match match = mb.build();
 511             FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "routerarp", match);
 512             FlowBuilder flowb = base().setPriority(150)
 513                 .setId(flowId)
 514                 .setMatch(match)
 515                 .setInstructions(
 516                         instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac),
 517                                 nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(),
 518                                 nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(),
 519                                 nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue()
 520                                         + ":INPORT")))));
 521             return flowb.build();
 522         } else {
 523             LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId()
 524                 .getValue());
 525             return null;
 526         }
 527
 528     }
 529
 530     private Flow createLocalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) {
 531
 532         // TODO Li alagalah - refactor common code but keep simple method
 533         ArrayList<Instruction> instructions = new ArrayList<>();
 534         List<Action> applyActions = new ArrayList<>();
 535
 536         int order = 0;
 537
 538         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 539         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 540         Action setNextHop;
 541         String nextHop;
 542
 543         // BEGIN L2 LOCAL
 544         nextHop = ofc.getNodeConnectorId().getValue();
 545
 546         long portNum;
 547         try {
 548             portNum = getOfPortNum(ofc.getNodeConnectorId());
 549         } catch (NumberFormatException ex) {
 550             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 551             return null;
 552         }
 553
 554         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 555
 556         // END L2 LOCAL
 557
 558         order += 1;
 559         applyActions.add(setdEPG);
 560         applyActions.add(setdCG);
 561         applyActions.add(setNextHop);
 562         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 563             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 564             .build();
 565         instructions.add(applyActionsIns);
 566
 567         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 568             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 569             .build();
 570         instructions.add(gotoTable);
 571
 572         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 573         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
 574         Match match = mb.build();
 575         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL2", match);
 576         FlowBuilder flowb = base().setId(flowid)
 577             .setPriority(Integer.valueOf(50))
 578             .setMatch(match)
 579             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 580         return flowb.build();
 581     }
 582
 583     private void syncEP(FlowMap flowMap, NodeId nodeId, PolicyInfo policyInfo, Endpoint srcEp, Endpoint destEp)
 584             throws Exception {
 585
 586         // TODO: Conditions messed up, but for now, send policyInfo until this
 587         // is fixed.
 588         EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, destEp);
 589         EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, srcEp);
 590
 591         if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) {
 592             LOG.trace("Didn't process endpoint due to either tenant, or EPG(s) being null", destEp.getKey());
 593             return;
 594         }
 595         OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class);
 596
 597         if (LocationType.External.equals(ofc.getLocationType())) {
 598             LOG.error("syncEp(): External endpoints should not be seen here.");
 599             return;
 600         }
 601
 602         /*
 603          * Only care about subnets for L3, but fetch them before loop. We need
 604          * the local subnets for setting SRC MAC for routing. All Routing is now
 605          * done locally! YAY! Instead of being shovelled L2 style across network
 606          * ala Helium.
 607          */
 608         List<Subnet> localSubnets = getLocalSubnets(nodeId);
 609         if (localSubnets == null) {
 610             LOG.error("No subnets could be found locally for node: {}", nodeId);
 611             return;
 612         }
 613
 614         if (Objects.equals(ofc.getNodeId(), nodeId)) {
 615             // this is a local endpoint; send to the approppriate local
 616             // port
 617
 618             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 619                 flowMap.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc));
 620             }
 621             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 622             // The Endpoint conflation must end!
 623             if (destEp.getL3Address() == null) {
 624                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 625                 return;
 626             }
 627
 628             for (L3Address l3a : destEp.getL3Address()) {
 629                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 630                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 631                             destEp.getL3Address());
 632                     continue;
 633                 } else {
 634                     for (Subnet localSubnet : localSubnets) {
 635                         Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet);
 636                         if (flow != null) {
 637                             flowMap.writeFlow(nodeId, TABLE_ID, flow);
 638                         } else {
 639                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 640                                     localSubnet.getIpPrefix().getValue());
 641                         }
 642                     }
 643                 }
 644             }
 645         } else {
 646             // this endpoint is on a different switch; send to the
 647             // appropriate tunnel
 648             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 649                 Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc);
 650                 if (remoteL2Flow != null) {
 651                     flowMap.writeFlow(nodeId, TABLE_ID, remoteL2Flow);
 652                 }
 653             } else {
 654                 LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}",
 655                         srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId());
 656             }
 657
 658             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 659             // The Endpoint conflation must end!
 660             if (destEp.getL3Address() == null) {
 661                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 662                 return;
 663             }
 664             for (L3Address l3a : destEp.getL3Address()) {
 665                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 666                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 667                             destEp.getL3Address());
 668                     continue;
 669                 } else {
 670                     for (Subnet localSubnet : localSubnets) {
 671                         Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds,
 672                                 destEpFwdCtxOrds, ofc, localSubnet);
 673                         if (remoteL3Flow != null) {
 674                             flowMap.writeFlow(nodeId, TABLE_ID, remoteL3Flow);
 675                         } else {
 676                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 677                                     localSubnet.getIpPrefix().getValue());
 678                         }
 679                     }
 680                 }
 681             }
 682         } // remote (tunnel)
 683
 684         // }
 685
 686     }
 687
 688     /*
 689      * ################################## DestMapper Flow methods
 690      * ##################################
 691      */
 692     private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
 693             OfOverlayContext ofc, Subnet srcSubnet) {
 694
 695         // TODO Li alagalah - refactor common code but keep simple method
 696
 697         Subnet destSubnet = null;
 698         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 699         if (subnets == null) {
 700             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 701             return null;
 702         }
 703         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 704         for (Subnet subnet : subnets) {
 705             // TODO Li alagalah add IPv6 support
 706             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 707                 destSubnet = subnet;
 708                 break;
 709             }
 710         }
 711         if (destSubnet == null) {
 712             LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 713             return null;
 714         }
 715
 716         if (destSubnet.getVirtualRouterIp() == null) {
 717             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 718                     destL3Address.getKey());
 719             return null;
 720         }
 721
 722         if (srcSubnet.getVirtualRouterIp() == null) {
 723             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 724             return null;
 725         }
 726         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 727         if (destL3c == null || destL3c.getId() == null) {
 728             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 729             return null;
 730         }
 731         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 732         if (srcL3c == null || srcL3c.getId() == null) {
 733             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 734             return null;
 735         }
 736
 737         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 738             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 739                 .getValue(), destL3c.getId().getValue());
 740             return null;
 741         }
 742
 743         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 744         MacAddress epDestMac = destEp.getMacAddress();
 745         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 746
 747         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 748             // This is our final destination, so match on actual EP mac.
 749             matcherMac = epDestMac;
 750         }
 751
 752         ArrayList<Instruction> l3instructions = new ArrayList<>();
 753         List<Action> applyActions = new ArrayList<>();
 754         List<Action> l3ApplyActions = new ArrayList<>();
 755
 756         int order = 0;
 757
 758         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 759         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 760         Action setNextHop;
 761         String nextHop;
 762
 763         // BEGIN L3 LOCAL
 764         nextHop = ofc.getNodeConnectorId().getValue();
 765
 766         long portNum;
 767         try {
 768             portNum = getOfPortNum(ofc.getNodeConnectorId());
 769         } catch (NumberFormatException ex) {
 770             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 771             return null;
 772         }
 773
 774         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 775         // END L3 LOCAL
 776
 777         // Lets not re-write the srcMac if its local.
 778         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
 779             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
 780             l3ApplyActions.add(setDlSrc);
 781         }
 782
 783         Action setDlDst = setDlDstAction(epDestMac);
 784         l3ApplyActions.add(setDlDst);
 785
 786         Action decTtl = decNwTtlAction();
 787         l3ApplyActions.add(decTtl);
 788
 789         order += 1;
 790         applyActions.add(setdEPG);
 791         applyActions.add(setdCG);
 792         applyActions.add(setNextHop);
 793
 794         applyActions.addAll(l3ApplyActions);
 795         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 796             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 797             .build();
 798
 799         l3instructions.add(applyActionsIns);
 800         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 801             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 802             .build();
 803         l3instructions.add(gotoTable);
 804         Layer3Match m = null;
 805         Long etherType = null;
 806         String ikey = null;
 807         if (destL3Address.getIpAddress().getIpv4Address() != null) {
 808             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
 809             etherType = IPv4;
 810             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 811         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
 812             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
 813             etherType = IPv6;
 814             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 815         } else {
 816             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
 817             return null;
 818         }
 819
 820         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
 821             .setLayer3Match(m);
 822         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 823         Match match = mb.build();
 824         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL3", match);
 825         FlowBuilder flowb = base().setId(flowid)
 826             .setPriority(Integer.valueOf(132))
 827             .setMatch(match)
 828             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 829         return flowb.build();
 830     }
 831
 832     private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds,
 833             EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) {
 834
 835         // TODO Li alagalah - refactor common code but keep simple method
 836
 837         // this endpoint is on a different switch; send to the
 838         // appropriate tunnel
 839
 840         ArrayList<Instruction> instructions = new ArrayList<>();
 841         List<Action> applyActions = new ArrayList<>();
 842
 843         int order = 0;
 844
 845         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
 846         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
 847         Action setNextHop;
 848         String nextHop;
 849
 850         // BEGIN TUNNEL HANDLING
 851         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
 852         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
 853         if (tunDst == null) {
 854             LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep);
 855             return null;
 856         }
 857         if (tunPort == null) {
 858             LOG.warn("Failed to get Tunnel Port for NodeId {} with EP {}", nodeId, ep);
 859             return null;
 860         }
 861
 862         Action tundstAction;
 863
 864         if (tunDst.getIpv4Address() != null) {
 865             nextHop = tunDst.getIpv4Address().getValue();
 866             tundstAction = nxLoadTunIPv4Action(nextHop, false);
 867         } else if (tunDst.getIpv6Address() != null) {
 868             // nextHop = tunDst.getIpv6Address().getValue();
 869             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
 870                     ofc.getNodeId());
 871             return null;
 872         } else {
 873             // this shouldn't happen
 874             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
 875             return null;
 876         }
 877
 878         long portNum;
 879         try {
 880             portNum = getOfPortNum(tunPort);
 881         } catch (NumberFormatException ex) {
 882             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 883             return null;
 884         }
 885
 886         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 887         applyActions.add(tundstAction);
 888         // END TUNNEL
 889
 890         order += 1;
 891         applyActions.add(setdEPG);
 892         applyActions.add(setdCG);
 893         applyActions.add(setNextHop);
 894         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 895             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 896             .build();
 897         instructions.add(applyActionsIns);
 898
 899         applyActionsIns = new InstructionBuilder().setOrder(order++)
 900             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 901             .build();
 902
 903         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 904             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 905             .build();
 906         instructions.add(gotoTable);
 907
 908         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 909         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId())));
 910         Match match = mb.build();
 911         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL2", match);
 912         FlowBuilder flowb = base().setId(flowid)
 913             .setPriority(Integer.valueOf(50))
 914             .setMatch(match)
 915             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 916
 917         return flowb.build();
 918     }
 919
 920     private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId,
 921             EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc,
 922             Subnet srcSubnet) {
 923
 924         // TODO Li alagalah - refactor common code but keep simple method
 925
 926         // this endpoint is on a different switch; send to the
 927         // appropriate tunnel
 928         Subnet destSubnet = null;
 929         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 930         if (subnets == null) {
 931             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 932             return null;
 933         }
 934         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 935         for (Subnet subnet : subnets) {
 936             // TODO Li alagalah add IPv6 support
 937             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 938                 destSubnet = subnet;
 939                 break;
 940             }
 941         }
 942         if (destSubnet == null) {
 943             LOG.info("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 944             return null;
 945         }
 946
 947         if (destSubnet.getVirtualRouterIp() == null) {
 948             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 949                     destL3Address.getKey());
 950             return null;
 951         }
 952
 953         if (srcSubnet.getVirtualRouterIp() == null) {
 954             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 955             return null;
 956         }
 957         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 958         if (destL3c == null || destL3c.getId() == null) {
 959             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 960             return null;
 961         }
 962         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 963         if (srcL3c == null || srcL3c.getId() == null) {
 964             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 965             return null;
 966         }
 967
 968         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 969             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 970                 .getValue(), destL3c.getId().getValue());
 971             return null;
 972         }
 973
 974         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 975         MacAddress epDestMac = destEp.getMacAddress();
 976         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 977         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 978             // This is our final destination, so match on actual EP mac.
 979             matcherMac = epDestMac;
 980         }
 981         ArrayList<Instruction> l3instructions = new ArrayList<>();
 982         List<Action> applyActions = new ArrayList<>();
 983         List<Action> l3ApplyActions = new ArrayList<>();
 984
 985         int order = 0;
 986
 987         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
 988         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
 989         Action setNextHop;
 990         String nextHop;
 991
 992         // BEGIN TUNNEL HANDLING
 993         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
 994         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
 995         if (tunDst == null) {
 996             LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address);
 997             return null;
 998         }
 999         if (tunPort == null) {
1000             LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, destL3Address);
1001             return null;
1002         }
1003
1004         Action tundstAction;
1005
1006         if (tunDst.getIpv4Address() != null) {
1007             nextHop = tunDst.getIpv4Address().getValue();
1008             tundstAction = nxLoadTunIPv4Action(nextHop, false);
1009         } else if (tunDst.getIpv6Address() != null) {
1010             // nextHop = tunDst.getIpv6Address().getValue();
1011             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
1012                     ofc.getNodeId());
1013             return null;
1014         } else {
1015             // this shouldn't happen
1016             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
1017             return null;
1018         }
1019
1020         long portNum;
1021         try {
1022             portNum = getOfPortNum(tunPort);
1023         } catch (NumberFormatException ex) {
1024             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
1025             return null;
1026         }
1027
1028         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1029         applyActions.add(tundstAction);
1030         // END TUNNEL
1031
1032         order += 1;
1033         applyActions.add(setdEPG);
1034         applyActions.add(setdCG);
1035         applyActions.add(setNextHop);
1036
1037         // Lets not re-write the srcMac if its local.
1038         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
1039             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
1040             l3ApplyActions.add(setDlSrc);
1041         }
1042
1043         Action setDlDst = setDlDstAction(epDestMac);
1044         l3ApplyActions.add(setDlDst);
1045
1046         Action decTtl = decNwTtlAction();
1047         l3ApplyActions.add(decTtl);
1048
1049         applyActions.addAll(l3ApplyActions);
1050         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1051             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1052             .build();
1053
1054         l3instructions.add(applyActionsIns);
1055         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1056             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1057             .build();
1058         l3instructions.add(gotoTable);
1059         Layer3Match m = null;
1060         Long etherType = null;
1061         String ikey = null;
1062         if (destL3Address.getIpAddress().getIpv4Address() != null) {
1063             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
1064             etherType = IPv4;
1065             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
1066         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
1067             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
1068             etherType = IPv6;
1069             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
1070         } else {
1071             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
1072             return null;
1073         }
1074
1075         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
1076             .setLayer3Match(m);
1077         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id())));
1078         Match match = mb.build();
1079         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL3", match);
1080         FlowBuilder flowb = base().setId(flowid)
1081             .setPriority(Integer.valueOf(132))
1082             .setMatch(match)
1083             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
1084         return flowb.build();
1085     }
1086
1087     private NetworkDomainId getEPNetworkContainment(Endpoint endpoint) {
1088         if (endpoint.getNetworkContainment() != null) {
1089             return endpoint.getNetworkContainment();
1090         } else {
1091             /*
1092              * TODO: Be alagalah: Endpoint Refactor: This should be set on input
1093              * which we can't do because of the backwards way endpoints were
1094              * "architected".
1095              */
1096             return ctx.getPolicyResolver()
1097                 .getTenant(endpoint.getTenant())
1098                 .getEndpointGroup(endpoint.getEndpointGroup())
1099                 .getNetworkDomain();
1100         }
1101     }
1102
1103     private HashSet<Subnet> getSubnets(final TenantId tenantId) {
1104
1105         // if (subnetsByTenant.get(tenantId) != null) {
1106         // return subnetsByTenant.get(tenantId);
1107         // }
1108
1109         if (ctx.getDataBroker() == null) {
1110             return null;
1111         }
1112
1113         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
1114         InstanceIdentifier<Tenant> tiid = TenantUtils.tenantIid(tenantId);
1115         Optional<Tenant> tenantInfo;
1116         try {
1117             tenantInfo = t.read(LogicalDatastoreType.CONFIGURATION, tiid).get();
1118         } catch (Exception e) {
1119             LOG.error("Could not read Tenant {}", tenantId, e);
1120             return null;
1121         }
1122
1123         HashSet<Subnet> subnets = new HashSet<Subnet>();
1124
1125         if (!tenantInfo.isPresent()) {
1126             LOG.warn("Tenant {} not found", tenantId);
1127             return null;
1128         }
1129
1130         subnets.addAll(tenantInfo.get().getSubnet());
1131         // subnetsByTenant.put(tenantId, subnets);
1132         return subnets;
1133     }
1134
1135     // Need a method to get subnets for EPs attached to the node locally
1136     // to set the source Mac address for the router interface.
1137     private List<Subnet> getLocalSubnets(NodeId nodeId) {
1138         Collection<Endpoint> endpointsForNode = ctx.getEndpointManager().getEndpointsForNode(nodeId);
1139
1140         List<Subnet> localSubnets = new ArrayList<Subnet>();
1141
1142         for (Endpoint endpoint : endpointsForNode) {
1143             HashSet<Subnet> subnets = getSubnets(endpoint.getTenant());
1144             if (subnets == null) {
1145                 LOG.error("No local subnets.");
1146                 return null;
1147             }
1148             NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint);
1149             for (Subnet subnet : subnets) {
1150                 if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) {
1151                     localSubnets.add(subnet);
1152                 }
1153             }
1154         }
1155         return localSubnets;
1156     }
1157
1158     static byte[] bytesFromHexString(String values) {
1159         String target = "";
1160         if (values != null) {
1161             target = values;
1162         }
1163         String[] octets = target.split(":");
1164
1165         byte[] ret = new byte[octets.length];
1166         for (int i = 0; i < octets.length; i++) {
1167             ret[i] = Integer.valueOf(octets[i], 16).byteValue();
1168         }
1169         return ret;
1170     }
1171 }