renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/mapper/destination/DestinationMapper.java

   1 /*
   2  * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.mapper.destination;
  10
  11 import static com.google.common.base.Preconditions.checkNotNull;
  12 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP;
  13 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4;
  14 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6;
  15 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch;
  16 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns;
  17 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction;
  18 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch;
  19 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum;
  20 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.gotoTableIns;
  21 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.groupAction;
  22 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.instructions;
  23 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpOpAction;
  24 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpShaAction;
  25 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction;
  26 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction;
  27 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action;
  28 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction;
  29 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction;
  30 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction;
  31 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction;
  32 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction;
  33 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction;
  34 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction;
  35 import static org.opendaylight.groupbasedpolicy.util.DataStoreHelper.readFromDs;
  36
  37 import java.math.BigInteger;
  38 import java.util.ArrayList;
  39 import java.util.Collection;
  40 import java.util.Collections;
  41 import java.util.HashMap;
  42 import java.util.HashSet;
  43 import java.util.List;
  44 import java.util.Map;
  45 import java.util.Map.Entry;
  46 import java.util.Objects;
  47 import java.util.Set;
  48
  49 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  50 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  51 import org.opendaylight.groupbasedpolicy.dto.EgKey;
  52 import org.opendaylight.groupbasedpolicy.dto.EpKey;
  53 import org.opendaylight.groupbasedpolicy.dto.IndexedTenant;
  54 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext;
  55 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfWriter;
  56 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.endpoint.EndpointManager;
  57 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowIdUtils;
  58 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowTable;
  59 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch;
  60 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory;
  61 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals;
  62 import org.opendaylight.groupbasedpolicy.util.IidFactory;
  63 import org.opendaylight.groupbasedpolicy.util.TenantUtils;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.go.to.table._case.GoToTable;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.SubnetId;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.l3.prefix.fields.EndpointL3Gateways;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Prefix;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.ForwardingContext;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L3Context;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.Subnet;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.ethernet.match.fields.EthernetDestinationBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatchBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.Layer3Match;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.dec.nw.ttl._case.DecNwTtl;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 110 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 111 import org.slf4j.Logger;
 112 import org.slf4j.LoggerFactory;
 113
 114 import com.google.common.base.Optional;
 115 import com.google.common.base.Strings;
 116 import com.google.common.collect.HashMultimap;
 117 import com.google.common.collect.SetMultimap;
 118 import com.google.common.collect.Sets;
 119
 120 /**
 121  * <h1>Manage the table that maps the destination address to the next hop for the
 122  * path as well as applies any relevant routing transformations (table=3)</h1>
 123  *
 124  * Sync Ep flows, every endpoint pair creates L2 and L3 flow<br>
 125  * <ul><li>Flow is external, when any {@link Endpoint} is external</li>
 126  * <li>Flow is local, when src and dst endpoint {@link EndpointFwdCtxOrdinals} are the same</li>
 127  * <li>Flow is local, when src and dst endpoint ordinals are not the same and {@link OfOverlayContext} is missing</li></ul>
 128  * Also applies to L3
 129  * <p>
 130  * L2 Flows:
 131  * <p>
 132  * <i>External, local and remote L2 flows</i><br>
 133  * Priority = 50<br>
 134  * Matches:<br>
 135  *      - dl_dst mac address {@link MacAddress}<br>
 136  *      - loadReg4 {@link NxmNxReg4}<br>
 137  * Actions:<br>
 138  *      - load tunnel Ipv4 (local and remote only)<br>
 139  *      - loadReg2 {@link NxmNxReg2}<br>
 140  *      - loadReg3 {@link NxmNxReg3}<br>
 141  *      - loadReg7 (next hop) {@link NxmNxReg7}<br>
 142  *      - {@link GoToTable} POLICY ENFORCER table<br>
 143  * <p>
 144  * L3 flows:
 145  * <p>
 146  * <i>External, local and remote L3 routed flows:</i><br>
 147  * Priority = 50<br>
 148  * Matches:<br>
 149  *      - ip (ethertype)
 150  *      - dl_dst mac address {@link MacAddress}<br>
 151  *      - setReg6 {@link NxmNxReg6}<br>
 152  * Actions:<br>
 153  *      - loadReg2 {@link NxmNxReg2}<br>
 154  *      - loadReg3 {@link NxmNxReg3}<br>
 155  *      - loadReg4 (tunnel destination) {@link NxmNxReg4} (remote only)<br>
 156  *      - loadReg7 (next hop) {@link NxmNxReg7}<br>
 157  *      - set dst mac to eth_dst {@link MacAddress}<br>
 158  *      - dec_ttl {@link DecNwTtl} (local only)<br>
 159  *      - {@link GoToTable} POLICY ENFORCER table
 160  * <p>
 161  * If virtual router ip is present in subnet, and subnet contains L3 context, arp flow is created<br>
 162  * <p>
 163  * <i>Router Arp flow</i><br>
 164  * Priority = 150<br>
 165  * Matches:<br>
 166  *      - arp (ethertype)<br>
 167  *      - arp target transport address<br>
 168  *      - setReg6 {@link NxmNxReg6}<br>
 169  * Actions:<br>
 170  *      - move eth_src = eth_dst<br>
 171  *      - set dl_src {@link MacAddress}<br>
 172  *      - load arp_op<br>
 173  *      - move arp_sha = arp_tha<br>
 174  *      - load arp_sha<br>
 175  *      - move arp_spa = arp_tpa<br>
 176  *      - load arp_spa<br>
 177  *      - output:port {@link NodeConnectorId}<br>
 178  * <p>
 179  * <i>Broadcast flow (per flood domain)</i>
 180  * Priority = 140<br>
 181  * Matches:<br>
 182  *      - ethernet destination {@link MacAddress}
 183  *      - setReg5 {@link NxmNxReg5}<br>
 184  * Actions:<br>
 185  *      - load tunnel ID<br>
 186  *      - group action<br>
 187  * <p>
 188  * <i>L3 Prefix flow</i><br>
 189  * Priority = 140<br>
 190  * Matches:<br>
 191  *      - ethernet destination {@link MacAddress}
 192  *      - setReg5 {@link NxmNxReg5}<br>
 193  * Actions:<br>
 194  *      - dl_dst {@link MacAddress}<br>
 195  *      - dec_ttl<br>
 196  *      - loadReg2 {@link NxmNxReg2}<br>
 197  *      - loadReg3 {@link NxmNxReg3}<br>
 198  *      - loadReg4 (next hop) {@link NxmNxReg4}<br>
 199  *      - loadReg7 (if internal, port_num == {@link NodeConnectorId of L2 EP} ) {@link NxmNxReg7}<br>
 200  *      - loadReg7 (if external, port_num = external port) {@link NxmNxReg7}<br>
 201  *      - {@link GoToTable} POLICY ENFORCER table
 202  */
 203 public class DestinationMapper extends FlowTable {
 204
 205     protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class);
 206
 207     // TODO Li alagalah: Improve UT coverage for this class.
 208
 209     // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows
 210     // This ensures we have the appropriate network-containment'
 211
 212     public static short TABLE_ID;
 213     /**
 214      * This is the MAC address of the magical router in the sky
 215      */
 216     public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5");
 217     public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00");
 218     public static final Integer BASE_L3_PRIORITY = 100;
 219
 220     public DestinationMapper(OfContext ctx, short tableId) {
 221         super(ctx);
 222         this.TABLE_ID = tableId;
 223     }
 224
 225     Map<TenantId, HashSet<Subnet>> subnetsByTenant = new HashMap<TenantId, HashSet<Subnet>>();
 226
 227     @Override
 228     public short getTableId() {
 229         return TABLE_ID;
 230     }
 231
 232     @Override
 233     public void sync(Endpoint endpoint, OfWriter ofWriter) throws Exception {
 234
 235         // TODO: only temporary workaround, use src & dst endpoint in implementation
 236         NodeId nodeId = ctx.getEndpointManager().getEndpointNodeId(endpoint);
 237
 238         TenantId currentTenant = null;
 239
 240         ofWriter.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null, TABLE_ID));
 241
 242         SetMultimap<EpKey, EpKey> visitedEps = HashMultimap.create();
 243         Set<EndpointFwdCtxOrdinals> epOrdSet = new HashSet<>();
 244
 245         for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) {
 246             Set<EndpointGroupId> srcEpgIds = new HashSet<>();
 247             if (srcEp.getEndpointGroup() != null)
 248                 srcEpgIds.add(srcEp.getEndpointGroup());
 249             if (srcEp.getEndpointGroups() != null)
 250                 srcEpgIds.addAll(srcEp.getEndpointGroups());
 251
 252             for (EndpointGroupId epgId : srcEpgIds) {
 253                 EgKey epg = new EgKey(srcEp.getTenant(), epgId);
 254                 Set<EgKey> peers = Sets.union(Collections.singleton(epg), ctx.getCurrentPolicy().getPeers(epg));
 255                 for (EgKey peer : peers) {
 256                     Collection<Endpoint> endpointsForGroup = new HashSet<>();
 257                     endpointsForGroup.addAll(ctx.getEndpointManager().getEndpointsForGroup(peer));
 258                     endpointsForGroup.addAll(ctx.getEndpointManager().getExtEpsNoLocForGroup(peer));
 259                     for (Endpoint peerEp : endpointsForGroup) {
 260                         currentTenant = peerEp.getTenant();
 261                         subnetsByTenant.put(currentTenant, getSubnets(currentTenant));
 262                         EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress());
 263                         EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress());
 264
 265                         if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) {
 266                             continue;
 267                         }
 268                         syncEP(ofWriter, nodeId, srcEp, peerEp);
 269                         visitedEps.put(srcEpKey, peerEpKey);
 270
 271                         // Process subnets and flood-domains for epPeer
 272                         EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx,
 273                                 peerEp);
 274                         if (epOrds == null) {
 275                             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", peerEp);
 276                             continue;
 277                         }
 278
 279                         epOrdSet.add(epOrds);
 280                     }
 281                 }
 282             }
 283         }
 284
 285         for (Entry<TenantId, HashSet<Subnet>> subnetEntry : subnetsByTenant.entrySet()) {
 286             if (subnetEntry.getValue() == null) {
 287                 LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey());
 288                 continue;
 289             }
 290             currentTenant = subnetEntry.getKey();
 291             for (Subnet sn : subnetEntry.getValue()) {
 292                 L3Context l3c = getL3ContextForSubnet(currentTenant, sn);
 293                 Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn,
 294                         OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId()));
 295                 if (arpFlow != null) {
 296                     ofWriter.writeFlow(nodeId, TABLE_ID, arpFlow);
 297                 } else {
 298                     LOG.debug(
 299                             "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .",
 300                             sn.getIpPrefix().getValue());
 301                 }
 302             }
 303         }
 304
 305         // Write broadcast flows per flood domain.
 306         for (EndpointFwdCtxOrdinals epOrd : epOrdSet) {
 307             if (ofWriter.groupExists(nodeId, Integer.valueOf(epOrd.getFdId()).longValue())) {
 308                 ofWriter.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd));
 309             }
 310         }
 311
 312         // L3 Prefix Endpoint handling
 313         Collection<EndpointL3Prefix> prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(currentTenant);
 314         if (prefixEps != null) {
 315             LOG.trace("DestinationMapper - Processing L3PrefixEndpoints");
 316             for (EndpointL3Prefix prefixEp : prefixEps) {
 317                 List<Subnet> localSubnets = getLocalSubnets(nodeId);
 318                 if (localSubnets == null) {
 319                     continue;
 320                 }
 321                 for (Subnet localSubnet: localSubnets) {
 322                     Flow prefixFlow = createL3PrefixFlow(prefixEp, nodeId, localSubnet);
 323                     if (prefixFlow != null) {
 324                         ofWriter.writeFlow(nodeId, TABLE_ID, prefixFlow);
 325                         LOG.trace("Wrote L3Prefix flow");
 326                     }
 327                 }
 328             }
 329         }
 330     }
 331
 332
 333
 334     // set up next-hop destinations for all the endpoints in the endpoint
 335     // group on the node
 336
 337     private Flow createL3PrefixFlow(EndpointL3Prefix prefixEp, NodeId nodeId, Subnet subnet) throws Exception {
 338         /*
 339          * Priority: 100+lengthprefix
 340          * Match: prefix, l3c, "mac address of router" ?
 341          * Action:
 342          * - set Reg2, Reg3 for L3Ep by L2Ep ?
 343          * - if external,
 344          * - Reg7: use switch location external port else punt for now
 345          * - if internal
 346          * - Reg7: grab L2Ep from L3Ep and use its location info
 347          * - goto_table: POLENF (will check there for external on EP)
 348          */
 349
 350         ReadOnlyTransaction rTx = ctx.getDataBroker().newReadOnlyTransaction();
 351         // TODO Bug #3440 Target: Be - should support for more than first gateway.
 352         EndpointL3Gateways l3Gateway = prefixEp.getEndpointL3Gateways().get(0);
 353         Optional<EndpointL3> optL3Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 354                 IidFactory.l3EndpointIid(l3Gateway.getL3Context(), l3Gateway.getIpAddress()), rTx);
 355         if (!optL3Ep.isPresent()) {
 356             LOG.error("createL3PrefixFlow - L3Endpoint gateway {} for L3Prefix {} not found.", l3Gateway, prefixEp);
 357             return null;
 358         }
 359         EndpointL3 l3Ep = optL3Ep.get();
 360         if (l3Ep.getL2Context() == null || l3Ep.getMacAddress() == null) {
 361             LOG.debug("L3 endpoint representing L3 gateway does not contain L2-context or MAC address. {}", l3Ep);
 362             return null;
 363         }
 364         Optional<Endpoint> optL2Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 365                 IidFactory.endpointIid(l3Ep.getL2Context(), l3Ep.getMacAddress()), rTx);
 366         if (!optL2Ep.isPresent()) {
 367             LOG.error("createL3PrefixFlow - L2Endpoint for L3Gateway {} not found.", l3Ep);
 368             return null;
 369         }
 370         Endpoint l2Ep = optL2Ep.get();
 371         EndpointFwdCtxOrdinals epFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, l2Ep);
 372         if (epFwdCtxOrds == null) {
 373             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", l2Ep);
 374             return null;
 375         }
 376
 377         NetworkDomainId epNetworkContainment = getEPNetworkContainment(l2Ep);
 378
 379         MacAddress epDestMac = l2Ep.getMacAddress();
 380         MacAddress destSubnetGatewayMac = l2Ep.getMacAddress();
 381         L3Context destL3c = getL3ContextForSubnet(prefixEp.getTenant(), subnet);
 382         if (destL3c == null || destL3c.getId() == null) {
 383             LOG.error("No L3 Context found associated with subnet {}", subnet.getId());
 384             return null;
 385         }
 386
 387         MacAddress matcherMac = routerPortMac(destL3c, subnet.getVirtualRouterIp());
 388
 389         ArrayList<Instruction> l3instructions = new ArrayList<>();
 390         List<Action> applyActions = new ArrayList<>();
 391         List<Action> l3ApplyActions = new ArrayList<>();
 392
 393         int order = 0;
 394
 395         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 396         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 397         Action setNextHop;
 398         String nextHop=null;
 399
 400         OfOverlayContext ofc = l2Ep.getAugmentation(OfOverlayContext.class);
 401
 402         long portNum = -1;
 403         if (EndpointManager.isInternal(l2Ep, ctx.getTenant(l2Ep.getTenant()).getExternalImplicitGroups())) {
 404             checkNotNull(ofc.getNodeConnectorId());
 405             nextHop = ofc.getNodeConnectorId().getValue();
 406             try {
 407                 portNum = getOfPortNum(ofc.getNodeConnectorId());
 408             } catch (NumberFormatException ex) {
 409                 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 410                 return null;
 411             }
 412
 413         } else {
 414             // External
 415             Set<NodeConnectorId> externalPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
 416             checkNotNull(externalPorts);
 417             for (NodeConnectorId externalPort : externalPorts) {
 418                 // TODO Bug #3440 Target: Be - should support for more than first external port.
 419                 //TODO Bug 3546 - Difficult: External port is unrelated to Tenant, L3C, L2BD..
 420                 nextHop = externalPort.getValue();
 421                 try {
 422                     portNum = getOfPortNum(externalPort);
 423                 } catch (NumberFormatException ex) {
 424                     LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 425                     return null;
 426                 }
 427                 continue;
 428             }
 429         }
 430
 431         if (Strings.isNullOrEmpty(nextHop)
 432                 || portNum == -1) {
 433             LOG.error("createL3Prefix - Cannot find nodeConnectorId for {} for Prefix: ", l2Ep, prefixEp);
 434             return null;
 435         }
 436         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 437
 438         Action setDlDst = setDlDstAction(epDestMac);
 439         l3ApplyActions.add(setDlDst);
 440
 441         Action decTtl = decNwTtlAction();
 442         l3ApplyActions.add(decTtl);
 443
 444         order += 1;
 445         applyActions.add(setdEPG);
 446         applyActions.add(setdCG);
 447         applyActions.add(setNextHop);
 448
 449         applyActions.addAll(l3ApplyActions);
 450         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 451             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 452             .build();
 453
 454         l3instructions.add(applyActionsIns);
 455         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 456             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 457             .build();
 458         l3instructions.add(gotoTable);
 459
 460         Layer3Match m = null;
 461         Long etherType = null;
 462         String ikey = null;
 463         Integer prefixLength=0;
 464         if (prefixEp.getIpPrefix().getIpv4Prefix() != null) {
 465             ikey = prefixEp.getIpPrefix().getIpv4Prefix().getValue();
 466             etherType = IPv4;
 467             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv4Prefix().getValue().split("/")[1]);
 468             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 469         } else if (prefixEp.getIpPrefix().getIpv6Prefix() != null) {
 470             ikey = prefixEp.getIpPrefix().getIpv6Prefix().getValue();
 471             etherType = IPv6;
 472             /*
 473              *  This will result in flows with priority between 100-228, but since its matching on IPv6 prefix as well
 474              *  this shouldn't pose and issue, as the priority is more important within the address space of the matcher,
 475              *  even though technically flows are processed in priority order.
 476              */
 477
 478             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv6Prefix().getValue().split("/")[1]);
 479             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 480         } else {
 481             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", prefixEp);
 482             return null;
 483         }
 484
 485         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType));
 486         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 487         Match match = mb.build();
 488         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "L3prefix", match);
 489         FlowBuilder flowb = base().setId(flowid)
 490             .setPriority(Integer.valueOf(BASE_L3_PRIORITY+prefixLength))
 491             .setMatch(match)
 492             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 493         return flowb.build();
 494     }
 495
 496     private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) {
 497         MatchBuilder mb = new MatchBuilder()
 498                 .setEthernetMatch(new EthernetMatchBuilder().setEthernetDestination(
 499                         new EthernetDestinationBuilder().setAddress(MULTICAST_MAC)
 500                                 .setMask(MULTICAST_MAC)
 501                                 .build()).build());
 502         addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId())));
 503
 504         Match match = mb.build();
 505         FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "broadcast", match);
 506         FlowBuilder flowb = base().setPriority(Integer.valueOf(140))
 507             .setId(flowId)
 508             .setMatch(match)
 509             .setInstructions(
 510                     instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false),
 511                             groupAction(Long.valueOf(epOrd.getFdId())))));
 512
 513         return flowb.build();
 514     }
 515
 516     private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) {
 517
 518         if (ctx.getDataBroker() == null) {
 519             return null;
 520         }
 521
 522         MacAddress defaultMacAddress = ROUTER_MAC;
 523
 524         EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId());
 525         InstanceIdentifier<EndpointL3> endpointsIid = InstanceIdentifier.builder(Endpoints.class)
 526             .child(EndpointL3.class, l3Key)
 527             .build();
 528         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 529
 530         Optional<EndpointL3> r;
 531         try {
 532             r = t.read(LogicalDatastoreType.OPERATIONAL, endpointsIid).get();
 533             if (!r.isPresent())
 534                 return defaultMacAddress;
 535             EndpointL3 epL3 = r.get();
 536             if (epL3.getMacAddress() == null) {
 537                 return defaultMacAddress;
 538             } else {
 539                 return epL3.getMacAddress();
 540             }
 541         } catch (Exception e) {
 542             LOG.error("Error reading EndpointL3 {}.{}", l3c, ipAddress, e);
 543             return null;
 544         }
 545     }
 546
 547     private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) {
 548         IndexedTenant indexedTenant = ctx.getTenant(tenantId);
 549         if (indexedTenant == null) {
 550             LOG.debug("Tenant {} is null, cannot get L3 context", tenantId);
 551             return null;
 552         }
 553         L3Context l3c = indexedTenant.resolveL3Context(sn.getId());
 554         return l3c;
 555     }
 556
 557     private Flow createRouterArpFlow(TenantId tenantId, NodeId nodeId, Subnet sn, int l3Id) {
 558         if (sn == null || sn.getVirtualRouterIp() == null) {
 559             LOG.trace("Didn't create routerArpFlow since either subnet or subnet virtual router was null");
 560             return null;
 561         }
 562         /*
 563          * TODO: Li alagalah: This should be new Yang "gateways" list as well,
 564          * that expresses the gateway and prefixes it is interface for. Should
 565          * also check for external.
 566          */
 567         if (sn.getVirtualRouterIp().getIpv4Address() != null) {
 568             String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue();
 569
 570             L3Context l3c = getL3ContextForSubnet(tenantId, sn);
 571             if (l3c == null) {
 572                 LOG.error("No L3 Context found associated with subnet {}", sn.getId());
 573             }
 574
 575             MacAddress routerMac = routerPortMac(l3c, sn.getVirtualRouterIp());
 576             if (routerMac == null) {
 577                 return null;
 578             }
 579
 580             BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue()));
 581
 582             MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match(
 583                     new ArpMatchBuilder().setArpOp(Integer.valueOf(1))
 584                         .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32"))
 585                         .build());
 586             addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id)));
 587
 588             Match match = mb.build();
 589             FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "routerarp", match);
 590             FlowBuilder flowb = base().setPriority(150)
 591                 .setId(flowId)
 592                 .setMatch(match)
 593                 .setInstructions(
 594                         instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac),
 595                                 nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(),
 596                                 nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(),
 597                                 nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue()
 598                                         + ":INPORT")))));
 599             return flowb.build();
 600         } else {
 601             LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId()
 602                 .getValue());
 603             return null;
 604         }
 605
 606     }
 607
 608     private Flow createLocalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) {
 609
 610         // TODO Li alagalah - refactor common code but keep simple method
 611         ArrayList<Instruction> instructions = new ArrayList<>();
 612         List<Action> applyActions = new ArrayList<>();
 613
 614         int order = 0;
 615
 616         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 617         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 618         Action setNextHop;
 619         String nextHop;
 620
 621         // BEGIN L2 LOCAL
 622         nextHop = ofc.getNodeConnectorId().getValue();
 623
 624         long portNum;
 625         try {
 626             portNum = getOfPortNum(ofc.getNodeConnectorId());
 627         } catch (NumberFormatException ex) {
 628             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 629             return null;
 630         }
 631
 632         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 633
 634         // END L2 LOCAL
 635
 636         order += 1;
 637         applyActions.add(setdEPG);
 638         applyActions.add(setdCG);
 639         applyActions.add(setNextHop);
 640         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 641             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 642             .build();
 643         instructions.add(applyActionsIns);
 644
 645         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 646             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 647             .build();
 648         instructions.add(gotoTable);
 649
 650         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 651         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
 652         Match match = mb.build();
 653         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL2", match);
 654         FlowBuilder flowb = base().setId(flowid)
 655             .setPriority(Integer.valueOf(50))
 656             .setMatch(match)
 657             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 658         return flowb.build();
 659     }
 660
 661     private void syncEP(OfWriter ofWriter, NodeId nodeId, Endpoint srcEp, Endpoint destEp)
 662             throws Exception {
 663
 664         if (ctx.getTenant(srcEp.getTenant()) == null
 665                 || ctx.getTenant(destEp.getTenant()) == null) {
 666             LOG.debug("Source or destination EP references empty tenant srcEp:{} destEp:{}", srcEp, destEp);
 667             return;
 668         }
 669
 670         // TODO: Conditions messed up, but for now, send policyInfo until this
 671         // is fixed.
 672         EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, destEp);
 673         if (destEpFwdCtxOrds == null) {
 674             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", destEp);
 675             return;
 676         }
 677         EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, srcEp);
 678         if (srcEpFwdCtxOrds == null) {
 679             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", srcEp);
 680             return;
 681         }
 682
 683
 684         if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) {
 685             if (destEp.getTenant() == null) {
 686                 LOG.debug("Didn't process endpoint {} due to tenant being null", destEp.getKey());
 687             } else {
 688                 LOG.debug("Didn't process endpoint {} due to EPG(s) being null", destEp.getKey());
 689             }
 690             return;
 691         }
 692
 693         /*
 694          * Only care about subnets for L3, but fetch them before loop. We need
 695          * the local subnets for setting SRC MAC for routing. All Routing is now
 696          * done locally! YAY! Instead of being shovelled L2 style across network
 697          * ala Helium.
 698          */
 699         List<Subnet> localSubnets = getLocalSubnets(nodeId);
 700         if (localSubnets == null) {
 701             LOG.error("No subnets could be found locally for node: {}", nodeId);
 702             return;
 703         }
 704
 705         OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class);
 706
 707         // forwarding outside of internal domain should be done when dest EP or GW is external.
 708         Subnet srcSubnet = ctx.getTenant(srcEp.getTenant()).resolveSubnet(new SubnetId(srcEp.getNetworkContainment()));
 709         Endpoint l2Gw = this.getL2EndpointOfSubnetGateway(srcEp.getTenant(), srcSubnet);
 710         boolean destEpIsExternal = destEp.getNetworkContainment() != null
 711                 && EndpointManager.isExternal(destEp, ctx.getTenant(destEp.getTenant()).getExternalImplicitGroups());
 712         boolean subnetGwIsExternal = l2Gw != null
 713                 && EndpointManager.isExternal(l2Gw, ctx.getTenant(l2Gw.getTenant()).getExternalImplicitGroups());
 714         if (destEpIsExternal || subnetGwIsExternal) {
 715             if (ofc == null && destEp.getNetworkContainment().equals(srcEp.getNetworkContainment())) {
 716                 Flow flow = createExternalL2Flow(destEp, destEpFwdCtxOrds, nodeId);
 717                 if (flow != null) {
 718                     ofWriter.writeFlow(nodeId, TABLE_ID, flow);
 719                 }
 720             } else if (l2Gw != null && EndpointManager.isExternal(l2Gw, ctx.getTenant(l2Gw.getTenant()).getExternalImplicitGroups())
 721                     && !destEp.getNetworkContainment().equals(srcEp.getNetworkContainment())) {
 722                 for (L3Address l3a : destEp.getL3Address()) {
 723                     if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 724                         LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 725                                 destEp.getL3Address());
 726                         continue;
 727                     }
 728                     for (Subnet localSubnet : localSubnets) {
 729                         Flow extL3Flow = createExternalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, localSubnet, nodeId);
 730                         if (extL3Flow != null) {
 731                             ofWriter.writeFlow(nodeId, TABLE_ID, extL3Flow);
 732                         } else {
 733                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 734                                     localSubnet.getIpPrefix().getValue());
 735                         }
 736                     }
 737                 }
 738             }
 739         }
 740         else if (ofc != null && Objects.equals(ofc.getNodeId(), nodeId)) {
 741             // this is a local endpoint; send to the approppriate local
 742             // port
 743
 744             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 745                 ofWriter.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc));
 746             }
 747             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 748             // The Endpoint conflation must end!
 749             if (destEp.getL3Address() == null) {
 750                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 751                 return;
 752             }
 753
 754             for (L3Address l3a : destEp.getL3Address()) {
 755                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 756                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 757                             destEp.getL3Address());
 758                     continue;
 759                 } else {
 760                     for (Subnet localSubnet : localSubnets) {
 761                         Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet);
 762                         if (flow != null) {
 763                             ofWriter.writeFlow(nodeId, TABLE_ID, flow);
 764                         } else {
 765                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 766                                     localSubnet.getIpPrefix().getValue());
 767                         }
 768                     }
 769                 }
 770             }
 771         } else if(ofc!= null) {
 772             // this endpoint is on a different switch; send to the
 773             // appropriate tunnel
 774             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 775                 Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc);
 776                 if (remoteL2Flow != null) {
 777                     ofWriter.writeFlow(nodeId, TABLE_ID, remoteL2Flow);
 778                 }
 779             } else {
 780                 LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}",
 781                         srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId());
 782             }
 783
 784             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 785             // The Endpoint conflation must end!
 786             if (destEp.getL3Address() == null) {
 787                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 788                 return;
 789             }
 790             for (L3Address l3a : destEp.getL3Address()) {
 791                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 792                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 793                             destEp.getL3Address());
 794                     continue;
 795                 } else {
 796                     for (Subnet localSubnet : localSubnets) {
 797                         Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds,
 798                                 destEpFwdCtxOrds, ofc, localSubnet);
 799                         if (remoteL3Flow != null) {
 800                             ofWriter.writeFlow(nodeId, TABLE_ID, remoteL3Flow);
 801                         } else {
 802                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 803                                     localSubnet.getIpPrefix().getValue());
 804                         }
 805                     }
 806                 }
 807             }
 808         } // remote (tunnel)
 809
 810         // }
 811
 812     }
 813
 814     /*
 815      * ################################## DestMapper Flow methods
 816      * ##################################
 817      */
 818     private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
 819             OfOverlayContext ofc, Subnet srcSubnet) {
 820
 821         // TODO Li alagalah - refactor common code but keep simple method
 822
 823         Subnet destSubnet = null;
 824         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 825         if (subnets == null) {
 826             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 827             return null;
 828         }
 829         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 830         for (Subnet subnet : subnets) {
 831             // TODO Li alagalah add IPv6 support
 832             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 833                 destSubnet = subnet;
 834                 break;
 835             }
 836         }
 837         if (destSubnet == null) {
 838             LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 839             return null;
 840         }
 841
 842         if (destSubnet.getVirtualRouterIp() == null) {
 843             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 844                     destL3Address.getKey());
 845             return null;
 846         }
 847
 848         if (srcSubnet.getVirtualRouterIp() == null) {
 849             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 850             return null;
 851         }
 852         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 853         if (destL3c == null || destL3c.getId() == null) {
 854             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 855             return null;
 856         }
 857         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 858         if (srcL3c == null || srcL3c.getId() == null) {
 859             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 860             return null;
 861         }
 862
 863         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 864             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 865                 .getValue(), destL3c.getId().getValue());
 866             return null;
 867         }
 868
 869         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 870         MacAddress epDestMac = destEp.getMacAddress();
 871         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 872
 873         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 874             // This is our final destination, so match on actual EP mac.
 875             matcherMac = epDestMac;
 876         }
 877
 878         ArrayList<Instruction> l3instructions = new ArrayList<>();
 879         List<Action> applyActions = new ArrayList<>();
 880         List<Action> l3ApplyActions = new ArrayList<>();
 881
 882         int order = 0;
 883
 884         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 885         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 886         Action setNextHop;
 887         String nextHop;
 888
 889         // BEGIN L3 LOCAL
 890         nextHop = ofc.getNodeConnectorId().getValue();
 891
 892         long portNum;
 893         try {
 894             portNum = getOfPortNum(ofc.getNodeConnectorId());
 895         } catch (NumberFormatException ex) {
 896             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 897             return null;
 898         }
 899
 900         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 901         // END L3 LOCAL
 902
 903         // Lets not re-write the srcMac if its local.
 904         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
 905             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
 906             l3ApplyActions.add(setDlSrc);
 907         }
 908
 909         Action setDlDst = setDlDstAction(epDestMac);
 910         l3ApplyActions.add(setDlDst);
 911
 912         Action decTtl = decNwTtlAction();
 913         l3ApplyActions.add(decTtl);
 914
 915         order += 1;
 916         applyActions.add(setdEPG);
 917         applyActions.add(setdCG);
 918         applyActions.add(setNextHop);
 919
 920         applyActions.addAll(l3ApplyActions);
 921         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 922             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 923             .build();
 924
 925         l3instructions.add(applyActionsIns);
 926         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 927             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 928             .build();
 929         l3instructions.add(gotoTable);
 930         Layer3Match m = null;
 931         Long etherType = null;
 932         String ikey = null;
 933         if (destL3Address.getIpAddress().getIpv4Address() != null) {
 934             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
 935             etherType = IPv4;
 936             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 937         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
 938             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
 939             etherType = IPv6;
 940             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 941         } else {
 942             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
 943             return null;
 944         }
 945
 946         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
 947             .setLayer3Match(m);
 948         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 949         Match match = mb.build();
 950         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL3", match);
 951         FlowBuilder flowb = base().setId(flowid)
 952             .setPriority(Integer.valueOf(132))
 953             .setMatch(match)
 954             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 955         return flowb.build();
 956     }
 957
 958     private Flow createExternalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
 959             Subnet srcSubnet, NodeId nodeId) {
 960
 961         Subnet destSubnet = null;
 962         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 963         if (subnets == null) {
 964             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 965             return null;
 966         }
 967         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 968         for (Subnet subnet : subnets) {
 969             // TODO Li alagalah add IPv6 support
 970             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 971                 destSubnet = subnet;
 972                 break;
 973             }
 974         }
 975         if (destSubnet == null) {
 976             LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 977             return null;
 978         }
 979
 980         if (destSubnet.getVirtualRouterIp() == null) {
 981             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 982                     destL3Address.getKey());
 983             return null;
 984         }
 985
 986         if (srcSubnet.getVirtualRouterIp() == null) {
 987             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 988             return null;
 989         }
 990         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 991         if (destL3c == null || destL3c.getId() == null) {
 992             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 993             return null;
 994         }
 995         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 996         if (srcL3c == null || srcL3c.getId() == null) {
 997             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 998             return null;
 999         }
1000
1001         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
1002             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
1003                 .getValue(), destL3c.getId().getValue());
1004             return null;
1005         }
1006
1007         Endpoint l2Gw = getL2EndpointOfSubnetGateway(destEp.getTenant(), srcSubnet);
1008         if(l2Gw == null) {
1009             LOG.warn("The endpoint representing external gateway of subnet {} not found", srcSubnet);
1010             return null;
1011         }
1012         MacAddress matcherMac = destEp.getMacAddress();
1013         MacAddress destSubnetGatewayMac = l2Gw.getMacAddress();
1014
1015         ArrayList<Instruction> l3instructions = new ArrayList<>();
1016         List<Action> applyActions = new ArrayList<>();
1017         List<Action> l3ApplyActions = new ArrayList<>();
1018
1019         int order = 0;
1020
1021         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
1022         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
1023         Action setNextHop;
1024
1025         Set<NodeConnectorId> extPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
1026         if (extPorts == null || !extPorts.iterator().hasNext()) {
1027             LOG.warn("No external interface on node: {}. External Gateway {} is not reachable!", nodeId, l2Gw.getKey());
1028             return null;
1029         }
1030         // only one external port is supported for now
1031          NodeConnectorId extPort = extPorts.iterator().next();
1032
1033         long portNum;
1034         try {
1035             portNum = getOfPortNum(extPort);
1036         } catch (NumberFormatException ex) {
1037             LOG.warn("Could not parse port number {}", extPort, ex);
1038             return null;
1039         }
1040
1041         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1042         // END L3 LOCAL
1043
1044
1045         Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
1046         l3ApplyActions.add(setDlSrc);
1047
1048         Action setDlDst = setDlDstAction(l2Gw.getMacAddress());
1049         l3ApplyActions.add(setDlDst);
1050
1051         order += 1;
1052         applyActions.add(setdEPG);
1053         applyActions.add(setdCG);
1054         applyActions.add(setNextHop);
1055
1056         applyActions.addAll(l3ApplyActions);
1057         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1058             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1059             .build();
1060
1061         l3instructions.add(applyActionsIns);
1062         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1063             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1064             .build();
1065         l3instructions.add(gotoTable);
1066         Layer3Match m = null;
1067         Long etherType = null;
1068         String ikey = null;
1069         if (destL3Address.getIpAddress().getIpv4Address() != null) {
1070             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
1071             etherType = IPv4;
1072             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
1073         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
1074             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
1075             etherType = IPv6;
1076             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
1077         } else {
1078             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
1079             return null;
1080         }
1081
1082         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
1083             .setLayer3Match(m);
1084         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
1085         Match match = mb.build();
1086         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "externalL3", match);
1087         FlowBuilder flowb = base().setId(flowid)
1088             .setPriority(Integer.valueOf(132))
1089             .setMatch(match)
1090             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
1091         return flowb.build();
1092     }
1093
1094     private Endpoint getL2EndpointOfSubnetGateway(TenantId tenantId, Subnet subnet) {
1095         if (subnet != null && subnet.getVirtualRouterIp() != null) {
1096             IpAddress gwIpAddress = subnet.getVirtualRouterIp();
1097             Collection<EndpointL3Prefix> prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(tenantId);
1098             if (prefixEps != null) {
1099                 for (EndpointL3Prefix prefixEp : prefixEps) {
1100                     for (EndpointL3Gateways gw : prefixEp.getEndpointL3Gateways()) {
1101                         EndpointL3 l3Ep = ctx.getEndpointManager().getL3Endpoint(gw.getL3Context(), gwIpAddress,
1102                                 prefixEp.getTenant());
1103                         if (l3Ep != null && l3Ep.getL2Context() != null && l3Ep.getMacAddress() != null) {
1104                             return ctx.getEndpointManager().getEndpoint(
1105                                     new EpKey(l3Ep.getL2Context(), l3Ep.getMacAddress()));
1106                         }
1107                     }
1108                 }
1109             }
1110         }
1111         return null;
1112     }
1113
1114     private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds,
1115             EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) {
1116
1117         // TODO Li alagalah - refactor common code but keep simple method
1118
1119         // this endpoint is on a different switch; send to the
1120         // appropriate tunnel
1121
1122         ArrayList<Instruction> instructions = new ArrayList<>();
1123         List<Action> applyActions = new ArrayList<>();
1124
1125         int order = 0;
1126
1127         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
1128         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
1129         Action setNextHop;
1130         String nextHop;
1131
1132         // BEGIN TUNNEL HANDLING
1133         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
1134         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
1135         if (tunDst == null) {
1136             LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep);
1137             return null;
1138         }
1139         if (tunPort == null) {
1140             LOG.warn("Failed to get Tunnel Port for NodeId {} with EP {}", nodeId, ep);
1141             return null;
1142         }
1143
1144         Action tundstAction;
1145
1146         if (tunDst.getIpv4Address() != null) {
1147             nextHop = tunDst.getIpv4Address().getValue();
1148             tundstAction = nxLoadTunIPv4Action(nextHop, false);
1149         } else if (tunDst.getIpv6Address() != null) {
1150             // nextHop = tunDst.getIpv6Address().getValue();
1151             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
1152                     ofc.getNodeId());
1153             return null;
1154         } else {
1155             // this shouldn't happen
1156             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
1157             return null;
1158         }
1159
1160         long portNum;
1161         try {
1162             portNum = getOfPortNum(tunPort);
1163         } catch (NumberFormatException ex) {
1164             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
1165             return null;
1166         }
1167
1168         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1169         applyActions.add(tundstAction);
1170         // END TUNNEL
1171
1172         order += 1;
1173         applyActions.add(setdEPG);
1174         applyActions.add(setdCG);
1175         applyActions.add(setNextHop);
1176         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1177             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1178             .build();
1179         instructions.add(applyActionsIns);
1180
1181         applyActionsIns = new InstructionBuilder().setOrder(order++)
1182             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1183             .build();
1184
1185         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1186             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1187             .build();
1188         instructions.add(gotoTable);
1189
1190         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
1191         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId())));
1192         Match match = mb.build();
1193         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL2", match);
1194         FlowBuilder flowb = base().setId(flowid)
1195             .setPriority(Integer.valueOf(50))
1196             .setMatch(match)
1197             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
1198
1199         return flowb.build();
1200     }
1201
1202     private Flow createExternalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds,NodeId nodeId) {
1203
1204         ArrayList<Instruction> instructions = new ArrayList<>();
1205         List<Action> applyActions = new ArrayList<>();
1206
1207         int order = 0;
1208
1209         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
1210         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
1211         Action setNextHop;
1212
1213         // BEGIN L2 LOCAL
1214         Set<NodeConnectorId> extPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
1215         if(extPorts == null || !extPorts.iterator().hasNext()) {
1216             return null;
1217         }
1218         // Only one external port is currently supported.
1219         NodeConnectorId extPort = extPorts.iterator().next();
1220         long portNum;
1221         try {
1222             portNum = getOfPortNum(extPort);
1223         } catch (NumberFormatException ex) {
1224             LOG.warn("Could not parse port number {}", extPort, ex);
1225             return null;
1226         }
1227         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1228
1229         // END L2 LOCAL
1230
1231         order += 1;
1232         applyActions.add(setdEPG);
1233         applyActions.add(setdCG);
1234         applyActions.add(setNextHop);
1235         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1236             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1237             .build();
1238         instructions.add(applyActionsIns);
1239
1240         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1241             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1242             .build();
1243         instructions.add(gotoTable);
1244
1245         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
1246         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
1247         Match match = mb.build();
1248         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "externalL2", match);
1249         FlowBuilder flowb = base().setId(flowid)
1250             .setPriority(Integer.valueOf(50))
1251             .setMatch(match)
1252             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
1253         return flowb.build();
1254     }
1255
1256     private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId,
1257             EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc,
1258             Subnet srcSubnet) {
1259
1260         // TODO Li alagalah - refactor common code but keep simple method
1261
1262         // this endpoint is on a different switch; send to the
1263         // appropriate tunnel
1264         Subnet destSubnet = null;
1265         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
1266         if (subnets == null) {
1267             LOG.trace("No subnets in tenant {}", destEp.getTenant());
1268             return null;
1269         }
1270         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
1271         for (Subnet subnet : subnets) {
1272             // TODO Li alagalah add IPv6 support
1273             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
1274                 destSubnet = subnet;
1275                 break;
1276             }
1277         }
1278         if (destSubnet == null) {
1279             LOG.info("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
1280             return null;
1281         }
1282
1283         if (destSubnet.getVirtualRouterIp() == null) {
1284             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
1285                     destL3Address.getKey());
1286             return null;
1287         }
1288
1289         if (srcSubnet.getVirtualRouterIp() == null) {
1290             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
1291             return null;
1292         }
1293         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
1294         if (destL3c == null || destL3c.getId() == null) {
1295             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
1296             return null;
1297         }
1298         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
1299         if (srcL3c == null || srcL3c.getId() == null) {
1300             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
1301             return null;
1302         }
1303
1304         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
1305             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
1306                 .getValue(), destL3c.getId().getValue());
1307             return null;
1308         }
1309
1310         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
1311         MacAddress epDestMac = destEp.getMacAddress();
1312         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
1313         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
1314             // This is our final destination, so match on actual EP mac.
1315             matcherMac = epDestMac;
1316         }
1317         ArrayList<Instruction> l3instructions = new ArrayList<>();
1318         List<Action> applyActions = new ArrayList<>();
1319         List<Action> l3ApplyActions = new ArrayList<>();
1320
1321         int order = 0;
1322
1323         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
1324         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
1325         Action setNextHop;
1326         String nextHop;
1327
1328         // BEGIN TUNNEL HANDLING
1329         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
1330         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
1331         if (tunDst == null) {
1332             LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address);
1333             return null;
1334         }
1335         if (tunPort == null) {
1336             LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, destL3Address);
1337             return null;
1338         }
1339
1340         Action tundstAction;
1341
1342         if (tunDst.getIpv4Address() != null) {
1343             nextHop = tunDst.getIpv4Address().getValue();
1344             tundstAction = nxLoadTunIPv4Action(nextHop, false);
1345         } else if (tunDst.getIpv6Address() != null) {
1346             // nextHop = tunDst.getIpv6Address().getValue();
1347             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
1348                     ofc.getNodeId());
1349             return null;
1350         } else {
1351             // this shouldn't happen
1352             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
1353             return null;
1354         }
1355
1356         long portNum;
1357         try {
1358             portNum = getOfPortNum(tunPort);
1359         } catch (NumberFormatException ex) {
1360             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
1361             return null;
1362         }
1363
1364         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1365         applyActions.add(tundstAction);
1366         // END TUNNEL
1367
1368         order += 1;
1369         applyActions.add(setdEPG);
1370         applyActions.add(setdCG);
1371         applyActions.add(setNextHop);
1372
1373         // Lets not re-write the srcMac if its local.
1374         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
1375             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
1376             l3ApplyActions.add(setDlSrc);
1377         }
1378
1379         Action setDlDst = setDlDstAction(epDestMac);
1380         l3ApplyActions.add(setDlDst);
1381
1382         Action decTtl = decNwTtlAction();
1383         l3ApplyActions.add(decTtl);
1384
1385         applyActions.addAll(l3ApplyActions);
1386         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1387             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1388             .build();
1389
1390         l3instructions.add(applyActionsIns);
1391         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1392             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1393             .build();
1394         l3instructions.add(gotoTable);
1395         Layer3Match m = null;
1396         Long etherType = null;
1397         String ikey = null;
1398         if (destL3Address.getIpAddress().getIpv4Address() != null) {
1399             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
1400             etherType = IPv4;
1401             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
1402         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
1403             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
1404             etherType = IPv6;
1405             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
1406         } else {
1407             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
1408             return null;
1409         }
1410
1411         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
1412             .setLayer3Match(m);
1413         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id())));
1414         Match match = mb.build();
1415         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL3", match);
1416         FlowBuilder flowb = base().setId(flowid)
1417             .setPriority(Integer.valueOf(132))
1418             .setMatch(match)
1419             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
1420         return flowb.build();
1421     }
1422
1423     private NetworkDomainId getEPNetworkContainment(Endpoint endpoint) {
1424         if (endpoint.getNetworkContainment() != null) {
1425             return endpoint.getNetworkContainment();
1426         } else {
1427             /*
1428              * TODO: Be alagalah: Endpoint Refactor: This should be set on input
1429              * which we can't do because of the backwards way endpoints were
1430              * "architected".
1431              */
1432             return ctx.getTenant(endpoint.getTenant())
1433                 .getEndpointGroup(endpoint.getEndpointGroup())
1434                 .getNetworkDomain();
1435         }
1436     }
1437
1438     private HashSet<Subnet> getSubnets(final TenantId tenantId) {
1439
1440         if (ctx.getDataBroker() == null) {
1441             return null;
1442         }
1443
1444         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
1445         InstanceIdentifier<Tenant> tiid = TenantUtils.tenantIid(tenantId);
1446         Optional<Tenant> tenantInfo;
1447         try {
1448             tenantInfo = t.read(LogicalDatastoreType.CONFIGURATION, tiid).get();
1449         } catch (Exception e) {
1450             LOG.error("Could not read Tenant {}", tenantId, e);
1451             return null;
1452         } finally {
1453             t.close();
1454         }
1455
1456         if (!tenantInfo.isPresent()) {
1457             LOG.warn("Tenant {} not found", tenantId);
1458             return null;
1459         }
1460
1461         ForwardingContext fwCtx = tenantInfo.get().getForwardingContext();
1462         if (fwCtx == null || fwCtx.getSubnet() == null) {
1463             return new HashSet<>();
1464         }
1465         return new HashSet<>(fwCtx.getSubnet());
1466     }
1467
1468     // Need a method to get subnets for EPs attached to the node locally
1469     // to set the source Mac address for the router interface.
1470     private List<Subnet> getLocalSubnets(NodeId nodeId) {
1471         Collection<Endpoint> endpointsForNode = ctx.getEndpointManager().getEndpointsForNode(nodeId);
1472
1473         List<Subnet> localSubnets = new ArrayList<Subnet>();
1474
1475         for (Endpoint endpoint : endpointsForNode) {
1476             HashSet<Subnet> subnets = getSubnets(endpoint.getTenant());
1477             if (subnets == null) {
1478                 LOG.debug("No local subnets in tenant {} for EP {}.", endpoint.getTenant(), endpoint.getKey());
1479                 continue;
1480             }
1481             NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint);
1482             for (Subnet subnet : subnets) {
1483                 if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) {
1484                     localSubnets.add(subnet);
1485                 }
1486             }
1487         }
1488         return localSubnets;
1489     }
1490
1491     public static byte[] bytesFromHexString(String values) {
1492         String target = "";
1493         if (values != null) {
1494             target = values;
1495         }
1496         String[] octets = target.split(":");
1497
1498         byte[] ret = new byte[octets.length];
1499         for (int i = 0; i < octets.length; i++) {
1500             ret[i] = Integer.valueOf(octets[i], 16).byteValue();
1501         }
1502         return ret;
1503     }
1504 }