renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java

   1 /*
   2  * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow;
  10
  11 import static com.google.common.base.Preconditions.checkNotNull;
  12 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP;
  13 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4;
  14 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6;
  15 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch;
  16 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns;
  17 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction;
  18 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch;
  19 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum;
  20 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.gotoTableIns;
  21 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.groupAction;
  22 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.instructions;
  23 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpOpAction;
  24 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpShaAction;
  25 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction;
  26 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction;
  27 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action;
  28 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction;
  29 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction;
  30 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction;
  31 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction;
  32 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction;
  33 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction;
  34 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction;
  35 import static org.opendaylight.groupbasedpolicy.util.DataStoreHelper.readFromDs;
  36
  37 import java.math.BigInteger;
  38 import java.util.ArrayList;
  39 import java.util.Collection;
  40 import java.util.Collections;
  41 import java.util.HashMap;
  42 import java.util.HashSet;
  43 import java.util.List;
  44 import java.util.Map;
  45 import java.util.Map.Entry;
  46 import java.util.Objects;
  47 import java.util.Set;
  48
  49 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  50 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  51 import org.opendaylight.groupbasedpolicy.dto.EgKey;
  52 import org.opendaylight.groupbasedpolicy.dto.EpKey;
  53 import org.opendaylight.groupbasedpolicy.dto.IndexedTenant;
  54 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext;
  55 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfWriter;
  56 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.endpoint.EndpointManager;
  57 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch;
  58 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals;
  59 import org.opendaylight.groupbasedpolicy.util.IidFactory;
  60 import org.opendaylight.groupbasedpolicy.util.TenantUtils;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
  63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.SubnetId;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.l3.prefix.fields.EndpointL3Gateways;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Prefix;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.ForwardingContext;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L2FloodDomain;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L3Context;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.Subnet;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.ethernet.match.fields.EthernetDestinationBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatchBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.Layer3Match;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 106 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 107 import org.slf4j.Logger;
 108 import org.slf4j.LoggerFactory;
 109
 110 import com.google.common.base.Optional;
 111 import com.google.common.base.Strings;
 112 import com.google.common.collect.HashMultimap;
 113 import com.google.common.collect.SetMultimap;
 114 import com.google.common.collect.Sets;
 115
 116 /**
 117  * Manage the table that maps the destination address to the next hop for the
 118  * path as well as applies any relevant routing transformations.
 119  */
 120 public class DestinationMapper extends FlowTable {
 121
 122     protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class);
 123
 124     // TODO Li alagalah: Improve UT coverage for this class.
 125
 126     // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows
 127     // This ensures we have the appropriate network-containment'
 128
 129     public static short TABLE_ID;
 130     /**
 131      * This is the MAC address of the magical router in the sky
 132      */
 133     public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5");
 134     public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00");
 135     public static final Integer BASE_L3_PRIORITY = 100;
 136
 137     public DestinationMapper(OfContext ctx, short tableId) {
 138         super(ctx);
 139         this.TABLE_ID = tableId;
 140     }
 141
 142     Map<TenantId, HashSet<Subnet>> subnetsByTenant = new HashMap<TenantId, HashSet<Subnet>>();
 143
 144     @Override
 145     public short getTableId() {
 146         return TABLE_ID;
 147     }
 148
 149     @Override
 150     public void sync(NodeId nodeId, OfWriter ofWriter) throws Exception {
 151
 152         TenantId currentTenant = null;
 153
 154         ofWriter.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null, TABLE_ID));
 155
 156         SetMultimap<EpKey, EpKey> visitedEps = HashMultimap.create();
 157         Set<EndpointFwdCtxOrdinals> epOrdSet = new HashSet<>();
 158
 159         for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) {
 160             Set<EndpointGroupId> srcEpgIds = new HashSet<>();
 161             if (srcEp.getEndpointGroup() != null)
 162                 srcEpgIds.add(srcEp.getEndpointGroup());
 163             if (srcEp.getEndpointGroups() != null)
 164                 srcEpgIds.addAll(srcEp.getEndpointGroups());
 165
 166             for (EndpointGroupId epgId : srcEpgIds) {
 167                 EgKey epg = new EgKey(srcEp.getTenant(), epgId);
 168                 Set<EgKey> peers = Sets.union(Collections.singleton(epg), ctx.getCurrentPolicy().getPeers(epg));
 169                 for (EgKey peer : peers) {
 170                     Collection<Endpoint> endpointsForGroup = new HashSet<>();
 171                     endpointsForGroup.addAll(ctx.getEndpointManager().getEndpointsForGroup(peer));
 172                     endpointsForGroup.addAll(ctx.getEndpointManager().getExtEpsNoLocForGroup(peer));
 173                     for (Endpoint peerEp : endpointsForGroup) {
 174                         currentTenant = peerEp.getTenant();
 175                         subnetsByTenant.put(currentTenant, getSubnets(currentTenant));
 176                         EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress());
 177                         EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress());
 178
 179                         if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) {
 180                             continue;
 181                         }
 182                         syncEP(ofWriter, nodeId, srcEp, peerEp);
 183                         visitedEps.put(srcEpKey, peerEpKey);
 184
 185                         // Process subnets and flood-domains for epPeer
 186                         EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx,
 187                                 peerEp);
 188                         if (epOrds == null) {
 189                             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", peerEp);
 190                             continue;
 191                         }
 192
 193                         epOrdSet.add(epOrds);
 194                     }
 195                 }
 196             }
 197         }
 198
 199         for (Entry<TenantId, HashSet<Subnet>> subnetEntry : subnetsByTenant.entrySet()) {
 200             if (subnetEntry.getValue() == null) {
 201                 LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey());
 202                 continue;
 203             }
 204             currentTenant = subnetEntry.getKey();
 205             for (Subnet sn : subnetEntry.getValue()) {
 206                 L3Context l3c = getL3ContextForSubnet(currentTenant, sn);
 207                 Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn,
 208                         OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId()));
 209                 if (arpFlow != null) {
 210                     ofWriter.writeFlow(nodeId, TABLE_ID, arpFlow);
 211                 } else {
 212                     LOG.debug(
 213                             "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .",
 214                             sn.getIpPrefix().getValue());
 215                 }
 216             }
 217         }
 218
 219         // Write broadcast flows per flood domain.
 220         for (EndpointFwdCtxOrdinals epOrd : epOrdSet) {
 221             if (ofWriter.groupExists(nodeId, Integer.valueOf(epOrd.getFdId()).longValue())) {
 222                 ofWriter.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd));
 223             }
 224         }
 225
 226         // L3 Prefix Endpoint handling
 227         Collection<EndpointL3Prefix> prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(currentTenant);
 228         if (prefixEps != null) {
 229             LOG.trace("DestinationMapper - Processing L3PrefixEndpoints");
 230             for (EndpointL3Prefix prefixEp : prefixEps) {
 231                 List<Subnet> localSubnets = getLocalSubnets(nodeId);
 232                 if (localSubnets == null) {
 233                     continue;
 234                 }
 235                 for (Subnet localSubnet: localSubnets) {
 236                     Flow prefixFlow = createL3PrefixFlow(prefixEp, nodeId, localSubnet);
 237                     if (prefixFlow != null) {
 238                         ofWriter.writeFlow(nodeId, TABLE_ID, prefixFlow);
 239                         LOG.trace("Wrote L3Prefix flow");
 240                     }
 241                 }
 242             }
 243         }
 244     }
 245
 246
 247
 248     // set up next-hop destinations for all the endpoints in the endpoint
 249     // group on the node
 250
 251     private Flow createL3PrefixFlow(EndpointL3Prefix prefixEp, NodeId nodeId, Subnet subnet) throws Exception {
 252         /*
 253          * Priority: 100+lengthprefix
 254          * Match: prefix, l3c, "mac address of router" ?
 255          * Action:
 256          * - set Reg2, Reg3 for L3Ep by L2Ep ?
 257          * - if external,
 258          * - Reg7: use switch location external port else punt for now
 259          * - if internal
 260          * - Reg7: grab L2Ep from L3Ep and use its location info
 261          * - goto_table: POLENF (will check there for external on EP)
 262          */
 263
 264         ReadOnlyTransaction rTx = ctx.getDataBroker().newReadOnlyTransaction();
 265         // TODO Bug #3440 Target: Be - should support for more than first gateway.
 266         EndpointL3Gateways l3Gateway = prefixEp.getEndpointL3Gateways().get(0);
 267         Optional<EndpointL3> optL3Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 268                 IidFactory.l3EndpointIid(l3Gateway.getL3Context(), l3Gateway.getIpAddress()), rTx);
 269         if (!optL3Ep.isPresent()) {
 270             LOG.error("createL3PrefixFlow - L3Endpoint gateway {} for L3Prefix {} not found.", l3Gateway, prefixEp);
 271             return null;
 272         }
 273         EndpointL3 l3Ep = optL3Ep.get();
 274         if (l3Ep.getL2Context() == null || l3Ep.getMacAddress() == null) {
 275             LOG.debug("L3 endpoint representing L3 gateway does not contain L2-context or MAC address. {}", l3Ep);
 276             return null;
 277         }
 278         Optional<Endpoint> optL2Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 279                 IidFactory.endpointIid(l3Ep.getL2Context(), l3Ep.getMacAddress()), rTx);
 280         if (!optL2Ep.isPresent()) {
 281             LOG.error("createL3PrefixFlow - L2Endpoint for L3Gateway {} not found.", l3Ep);
 282             return null;
 283         }
 284         Endpoint l2Ep = optL2Ep.get();
 285         EndpointFwdCtxOrdinals epFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, l2Ep);
 286         if (epFwdCtxOrds == null) {
 287             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", l2Ep);
 288             return null;
 289         }
 290
 291         NetworkDomainId epNetworkContainment = getEPNetworkContainment(l2Ep);
 292
 293         MacAddress epDestMac = l2Ep.getMacAddress();
 294         MacAddress destSubnetGatewayMac = l2Ep.getMacAddress();
 295         L3Context destL3c = getL3ContextForSubnet(prefixEp.getTenant(), subnet);
 296         if (destL3c == null || destL3c.getId() == null) {
 297             LOG.error("No L3 Context found associated with subnet {}", subnet.getId());
 298             return null;
 299         }
 300
 301         MacAddress matcherMac = routerPortMac(destL3c, subnet.getVirtualRouterIp());
 302
 303         ArrayList<Instruction> l3instructions = new ArrayList<>();
 304         List<Action> applyActions = new ArrayList<>();
 305         List<Action> l3ApplyActions = new ArrayList<>();
 306
 307         int order = 0;
 308
 309         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 310         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 311         Action setNextHop;
 312         String nextHop=null;
 313
 314         OfOverlayContext ofc = l2Ep.getAugmentation(OfOverlayContext.class);
 315
 316         long portNum = -1;
 317         if (EndpointManager.isInternal(l2Ep, ctx.getTenant(l2Ep.getTenant()).getExternalImplicitGroups())) {
 318             checkNotNull(ofc.getNodeConnectorId());
 319             nextHop = ofc.getNodeConnectorId().getValue();
 320             try {
 321                 portNum = getOfPortNum(ofc.getNodeConnectorId());
 322             } catch (NumberFormatException ex) {
 323                 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 324                 return null;
 325             }
 326
 327         } else {
 328             // External
 329             Set<NodeConnectorId> externalPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
 330             checkNotNull(externalPorts);
 331             for (NodeConnectorId externalPort : externalPorts) {
 332                 // TODO Bug #3440 Target: Be - should support for more than first external port.
 333                 //TODO Bug 3546 - Difficult: External port is unrelated to Tenant, L3C, L2BD..
 334                 nextHop = externalPort.getValue();
 335                 try {
 336                     portNum = getOfPortNum(externalPort);
 337                 } catch (NumberFormatException ex) {
 338                     LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 339                     return null;
 340                 }
 341                 continue;
 342             }
 343         }
 344
 345         if (Strings.isNullOrEmpty(nextHop)
 346                 || portNum == -1) {
 347             LOG.error("createL3Prefix - Cannot find nodeConnectorId for {} for Prefix: ", l2Ep, prefixEp);
 348             return null;
 349         }
 350         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 351
 352         Action setDlDst = setDlDstAction(epDestMac);
 353         l3ApplyActions.add(setDlDst);
 354
 355         Action decTtl = decNwTtlAction();
 356         l3ApplyActions.add(decTtl);
 357
 358         order += 1;
 359         applyActions.add(setdEPG);
 360         applyActions.add(setdCG);
 361         applyActions.add(setNextHop);
 362
 363         applyActions.addAll(l3ApplyActions);
 364         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 365             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 366             .build();
 367
 368         l3instructions.add(applyActionsIns);
 369         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 370             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 371             .build();
 372         l3instructions.add(gotoTable);
 373
 374         Layer3Match m = null;
 375         Long etherType = null;
 376         String ikey = null;
 377         Integer prefixLength=0;
 378         if (prefixEp.getIpPrefix().getIpv4Prefix() != null) {
 379             ikey = prefixEp.getIpPrefix().getIpv4Prefix().getValue();
 380             etherType = IPv4;
 381             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv4Prefix().getValue().split("/")[1]);
 382             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 383         } else if (prefixEp.getIpPrefix().getIpv6Prefix() != null) {
 384             ikey = prefixEp.getIpPrefix().getIpv6Prefix().getValue();
 385             etherType = IPv6;
 386             /*
 387              *  This will result in flows with priority between 100-228, but since its matching on IPv6 prefix as well
 388              *  this shouldn't pose and issue, as the priority is more important within the address space of the matcher,
 389              *  even though technically flows are processed in priority order.
 390              */
 391
 392             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv6Prefix().getValue().split("/")[1]);
 393             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 394         } else {
 395             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", prefixEp);
 396             return null;
 397         }
 398
 399         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType));
 400         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 401         Match match = mb.build();
 402         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "L3prefix", match);
 403         FlowBuilder flowb = base().setId(flowid)
 404             .setPriority(Integer.valueOf(BASE_L3_PRIORITY+prefixLength))
 405             .setMatch(match)
 406             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 407         return flowb.build();
 408     }
 409
 410     private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) {
 411         MatchBuilder mb = new MatchBuilder()
 412                 .setEthernetMatch(new EthernetMatchBuilder().setEthernetDestination(
 413                         new EthernetDestinationBuilder().setAddress(MULTICAST_MAC)
 414                                 .setMask(MULTICAST_MAC)
 415                                 .build()).build());
 416         addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId())));
 417
 418         Match match = mb.build();
 419         FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "broadcast", match);
 420         FlowBuilder flowb = base().setPriority(Integer.valueOf(140))
 421             .setId(flowId)
 422             .setMatch(match)
 423             .setInstructions(
 424                     instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false),
 425                             groupAction(Long.valueOf(epOrd.getFdId())))));
 426
 427         return flowb.build();
 428     }
 429
 430     private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) {
 431
 432         if (ctx.getDataBroker() == null) {
 433             return null;
 434         }
 435
 436         MacAddress defaultMacAddress = ROUTER_MAC;
 437
 438         EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId());
 439         InstanceIdentifier<EndpointL3> endpointsIid = InstanceIdentifier.builder(Endpoints.class)
 440             .child(EndpointL3.class, l3Key)
 441             .build();
 442         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 443
 444         Optional<EndpointL3> r;
 445         try {
 446             r = t.read(LogicalDatastoreType.OPERATIONAL, endpointsIid).get();
 447             if (!r.isPresent())
 448                 return defaultMacAddress;
 449             EndpointL3 epL3 = r.get();
 450             if (epL3.getMacAddress() == null) {
 451                 return defaultMacAddress;
 452             } else {
 453                 return epL3.getMacAddress();
 454             }
 455         } catch (Exception e) {
 456             LOG.error("Error reading EndpointL3 {}.{}", l3c, ipAddress, e);
 457             return null;
 458         }
 459     }
 460
 461     private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) {
 462         IndexedTenant indexedTenant = ctx.getTenant(tenantId);
 463         if (indexedTenant == null) {
 464             LOG.debug("Tenant {} is null, cannot get L3 context", tenantId);
 465             return null;
 466         }
 467         L3Context l3c = indexedTenant.resolveL3Context(sn.getId());
 468         return l3c;
 469     }
 470
 471     private Flow createRouterArpFlow(TenantId tenantId, NodeId nodeId, Subnet sn, int l3Id) {
 472         if (sn == null || sn.getVirtualRouterIp() == null) {
 473             LOG.trace("Didn't create routerArpFlow since either subnet or subnet virtual router was null");
 474             return null;
 475         }
 476         /*
 477          * TODO: Li alagalah: This should be new Yang "gateways" list as well,
 478          * that expresses the gateway and prefixes it is interface for. Should
 479          * also check for external.
 480          */
 481         if (sn.getVirtualRouterIp().getIpv4Address() != null) {
 482             String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue();
 483
 484             L3Context l3c = getL3ContextForSubnet(tenantId, sn);
 485             if (l3c == null) {
 486                 LOG.error("No L3 Context found associated with subnet {}", sn.getId());
 487             }
 488
 489             MacAddress routerMac = routerPortMac(l3c, sn.getVirtualRouterIp());
 490             if (routerMac == null) {
 491                 return null;
 492             }
 493
 494             BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue()));
 495
 496             MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match(
 497                     new ArpMatchBuilder().setArpOp(Integer.valueOf(1))
 498                         .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32"))
 499                         .build());
 500             addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id)));
 501
 502             Match match = mb.build();
 503             FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "routerarp", match);
 504             FlowBuilder flowb = base().setPriority(150)
 505                 .setId(flowId)
 506                 .setMatch(match)
 507                 .setInstructions(
 508                         instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac),
 509                                 nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(),
 510                                 nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(),
 511                                 nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue()
 512                                         + ":INPORT")))));
 513             return flowb.build();
 514         } else {
 515             LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId()
 516                 .getValue());
 517             return null;
 518         }
 519
 520     }
 521
 522     private Flow createLocalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) {
 523
 524         // TODO Li alagalah - refactor common code but keep simple method
 525         ArrayList<Instruction> instructions = new ArrayList<>();
 526         List<Action> applyActions = new ArrayList<>();
 527
 528         int order = 0;
 529
 530         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 531         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 532         Action setNextHop;
 533         String nextHop;
 534
 535         // BEGIN L2 LOCAL
 536         nextHop = ofc.getNodeConnectorId().getValue();
 537
 538         long portNum;
 539         try {
 540             portNum = getOfPortNum(ofc.getNodeConnectorId());
 541         } catch (NumberFormatException ex) {
 542             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 543             return null;
 544         }
 545
 546         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 547
 548         // END L2 LOCAL
 549
 550         order += 1;
 551         applyActions.add(setdEPG);
 552         applyActions.add(setdCG);
 553         applyActions.add(setNextHop);
 554         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 555             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 556             .build();
 557         instructions.add(applyActionsIns);
 558
 559         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 560             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 561             .build();
 562         instructions.add(gotoTable);
 563
 564         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 565         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
 566         Match match = mb.build();
 567         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL2", match);
 568         FlowBuilder flowb = base().setId(flowid)
 569             .setPriority(Integer.valueOf(50))
 570             .setMatch(match)
 571             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 572         return flowb.build();
 573     }
 574
 575     private void syncEP(OfWriter ofWriter, NodeId nodeId, Endpoint srcEp, Endpoint destEp)
 576             throws Exception {
 577
 578         if (ctx.getTenant(srcEp.getTenant()) == null
 579                 || ctx.getTenant(destEp.getTenant()) == null) {
 580             LOG.debug("Source or destination EP references empty tenant srcEp:{} destEp:{}", srcEp, destEp);
 581             return;
 582         }
 583
 584         // TODO: Conditions messed up, but for now, send policyInfo until this
 585         // is fixed.
 586         EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, destEp);
 587         if (destEpFwdCtxOrds == null) {
 588             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", destEp);
 589             return;
 590         }
 591         EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, srcEp);
 592         if (srcEpFwdCtxOrds == null) {
 593             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", srcEp);
 594             return;
 595         }
 596
 597
 598         if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) {
 599             if (destEp.getTenant() == null) {
 600                 LOG.debug("Didn't process endpoint {} due to tenant being null", destEp.getKey());
 601             } else {
 602                 LOG.debug("Didn't process endpoint {} due to EPG(s) being null", destEp.getKey());
 603             }
 604             return;
 605         }
 606
 607         /*
 608          * Only care about subnets for L3, but fetch them before loop. We need
 609          * the local subnets for setting SRC MAC for routing. All Routing is now
 610          * done locally! YAY! Instead of being shovelled L2 style across network
 611          * ala Helium.
 612          */
 613         List<Subnet> localSubnets = getLocalSubnets(nodeId);
 614         if (localSubnets == null) {
 615             LOG.error("No subnets could be found locally for node: {}", nodeId);
 616             return;
 617         }
 618
 619         OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class);
 620
 621         // forwarding outside of internal domain should be done when dest EP or GW is external.
 622         Subnet srcSubnet = ctx.getTenant(srcEp.getTenant()).resolveSubnet(new SubnetId(srcEp.getNetworkContainment()));
 623         Endpoint l2Gw = this.getL2EndpointOfSubnetGateway(srcEp.getTenant(), srcSubnet);
 624         boolean destEpIsExternal = destEp.getNetworkContainment() != null
 625                 && EndpointManager.isExternal(destEp, ctx.getTenant(destEp.getTenant()).getExternalImplicitGroups());
 626         boolean subnetGwIsExternal = l2Gw != null
 627                 && EndpointManager.isExternal(l2Gw, ctx.getTenant(l2Gw.getTenant()).getExternalImplicitGroups());
 628         if (destEpIsExternal || subnetGwIsExternal) {
 629             if (ofc == null && destEp.getNetworkContainment().equals(srcEp.getNetworkContainment())) {
 630                 Flow flow = createExternalL2Flow(destEp, destEpFwdCtxOrds, nodeId);
 631                 if (flow != null) {
 632                     ofWriter.writeFlow(nodeId, TABLE_ID, flow);
 633                 }
 634             } else if (l2Gw != null && EndpointManager.isExternal(l2Gw, ctx.getTenant(l2Gw.getTenant()).getExternalImplicitGroups())
 635                     && !destEp.getNetworkContainment().equals(srcEp.getNetworkContainment())) {
 636                 for (L3Address l3a : destEp.getL3Address()) {
 637                     if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 638                         LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 639                                 destEp.getL3Address());
 640                         continue;
 641                     }
 642                     for (Subnet localSubnet : localSubnets) {
 643                         Flow extL3Flow = createExternalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, localSubnet, nodeId);
 644                         if (extL3Flow != null) {
 645                             ofWriter.writeFlow(nodeId, TABLE_ID, extL3Flow);
 646                         } else {
 647                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 648                                     localSubnet.getIpPrefix().getValue());
 649                         }
 650                     }
 651                 }
 652             }
 653         }
 654         else if (ofc != null && Objects.equals(ofc.getNodeId(), nodeId)) {
 655             // this is a local endpoint; send to the approppriate local
 656             // port
 657
 658             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 659                 ofWriter.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc));
 660             }
 661             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 662             // The Endpoint conflation must end!
 663             if (destEp.getL3Address() == null) {
 664                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 665                 return;
 666             }
 667
 668             for (L3Address l3a : destEp.getL3Address()) {
 669                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 670                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 671                             destEp.getL3Address());
 672                     continue;
 673                 } else {
 674                     for (Subnet localSubnet : localSubnets) {
 675                         Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet);
 676                         if (flow != null) {
 677                             ofWriter.writeFlow(nodeId, TABLE_ID, flow);
 678                         } else {
 679                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 680                                     localSubnet.getIpPrefix().getValue());
 681                         }
 682                     }
 683                 }
 684             }
 685         } else if(ofc!= null) {
 686             // this endpoint is on a different switch; send to the
 687             // appropriate tunnel
 688             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 689                 Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc);
 690                 if (remoteL2Flow != null) {
 691                     ofWriter.writeFlow(nodeId, TABLE_ID, remoteL2Flow);
 692                 }
 693             } else {
 694                 LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}",
 695                         srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId());
 696             }
 697
 698             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 699             // The Endpoint conflation must end!
 700             if (destEp.getL3Address() == null) {
 701                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 702                 return;
 703             }
 704             for (L3Address l3a : destEp.getL3Address()) {
 705                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 706                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 707                             destEp.getL3Address());
 708                     continue;
 709                 } else {
 710                     for (Subnet localSubnet : localSubnets) {
 711                         Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds,
 712                                 destEpFwdCtxOrds, ofc, localSubnet);
 713                         if (remoteL3Flow != null) {
 714                             ofWriter.writeFlow(nodeId, TABLE_ID, remoteL3Flow);
 715                         } else {
 716                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 717                                     localSubnet.getIpPrefix().getValue());
 718                         }
 719                     }
 720                 }
 721             }
 722         } // remote (tunnel)
 723
 724         // }
 725
 726     }
 727
 728     /*
 729      * ################################## DestMapper Flow methods
 730      * ##################################
 731      */
 732     private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
 733             OfOverlayContext ofc, Subnet srcSubnet) {
 734
 735         // TODO Li alagalah - refactor common code but keep simple method
 736
 737         Subnet destSubnet = null;
 738         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 739         if (subnets == null) {
 740             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 741             return null;
 742         }
 743         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 744         for (Subnet subnet : subnets) {
 745             // TODO Li alagalah add IPv6 support
 746             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 747                 destSubnet = subnet;
 748                 break;
 749             }
 750         }
 751         if (destSubnet == null) {
 752             LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 753             return null;
 754         }
 755
 756         if (destSubnet.getVirtualRouterIp() == null) {
 757             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 758                     destL3Address.getKey());
 759             return null;
 760         }
 761
 762         if (srcSubnet.getVirtualRouterIp() == null) {
 763             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 764             return null;
 765         }
 766         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 767         if (destL3c == null || destL3c.getId() == null) {
 768             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 769             return null;
 770         }
 771         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 772         if (srcL3c == null || srcL3c.getId() == null) {
 773             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 774             return null;
 775         }
 776
 777         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 778             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 779                 .getValue(), destL3c.getId().getValue());
 780             return null;
 781         }
 782
 783         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 784         MacAddress epDestMac = destEp.getMacAddress();
 785         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 786
 787         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 788             // This is our final destination, so match on actual EP mac.
 789             matcherMac = epDestMac;
 790         }
 791
 792         ArrayList<Instruction> l3instructions = new ArrayList<>();
 793         List<Action> applyActions = new ArrayList<>();
 794         List<Action> l3ApplyActions = new ArrayList<>();
 795
 796         int order = 0;
 797
 798         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 799         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 800         Action setNextHop;
 801         String nextHop;
 802
 803         // BEGIN L3 LOCAL
 804         nextHop = ofc.getNodeConnectorId().getValue();
 805
 806         long portNum;
 807         try {
 808             portNum = getOfPortNum(ofc.getNodeConnectorId());
 809         } catch (NumberFormatException ex) {
 810             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 811             return null;
 812         }
 813
 814         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 815         // END L3 LOCAL
 816
 817         // Lets not re-write the srcMac if its local.
 818         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
 819             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
 820             l3ApplyActions.add(setDlSrc);
 821         }
 822
 823         Action setDlDst = setDlDstAction(epDestMac);
 824         l3ApplyActions.add(setDlDst);
 825
 826         Action decTtl = decNwTtlAction();
 827         l3ApplyActions.add(decTtl);
 828
 829         order += 1;
 830         applyActions.add(setdEPG);
 831         applyActions.add(setdCG);
 832         applyActions.add(setNextHop);
 833
 834         applyActions.addAll(l3ApplyActions);
 835         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 836             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 837             .build();
 838
 839         l3instructions.add(applyActionsIns);
 840         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 841             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 842             .build();
 843         l3instructions.add(gotoTable);
 844         Layer3Match m = null;
 845         Long etherType = null;
 846         String ikey = null;
 847         if (destL3Address.getIpAddress().getIpv4Address() != null) {
 848             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
 849             etherType = IPv4;
 850             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 851         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
 852             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
 853             etherType = IPv6;
 854             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 855         } else {
 856             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
 857             return null;
 858         }
 859
 860         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
 861             .setLayer3Match(m);
 862         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 863         Match match = mb.build();
 864         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL3", match);
 865         FlowBuilder flowb = base().setId(flowid)
 866             .setPriority(Integer.valueOf(132))
 867             .setMatch(match)
 868             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 869         return flowb.build();
 870     }
 871
 872     private Flow createExternalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
 873             Subnet srcSubnet, NodeId nodeId) {
 874
 875         Subnet destSubnet = null;
 876         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 877         if (subnets == null) {
 878             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 879             return null;
 880         }
 881         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 882         for (Subnet subnet : subnets) {
 883             // TODO Li alagalah add IPv6 support
 884             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 885                 destSubnet = subnet;
 886                 break;
 887             }
 888         }
 889         if (destSubnet == null) {
 890             LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 891             return null;
 892         }
 893
 894         if (destSubnet.getVirtualRouterIp() == null) {
 895             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 896                     destL3Address.getKey());
 897             return null;
 898         }
 899
 900         if (srcSubnet.getVirtualRouterIp() == null) {
 901             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 902             return null;
 903         }
 904         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 905         if (destL3c == null || destL3c.getId() == null) {
 906             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 907             return null;
 908         }
 909         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 910         if (srcL3c == null || srcL3c.getId() == null) {
 911             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 912             return null;
 913         }
 914
 915         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 916             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 917                 .getValue(), destL3c.getId().getValue());
 918             return null;
 919         }
 920
 921         Endpoint l2Gw = getL2EndpointOfSubnetGateway(destEp.getTenant(), srcSubnet);
 922         if(l2Gw == null) {
 923             LOG.warn("The endpoint representing external gateway of subnet {} not found", srcSubnet);
 924             return null;
 925         }
 926         MacAddress matcherMac = destEp.getMacAddress();
 927         MacAddress destSubnetGatewayMac = l2Gw.getMacAddress();
 928
 929         ArrayList<Instruction> l3instructions = new ArrayList<>();
 930         List<Action> applyActions = new ArrayList<>();
 931         List<Action> l3ApplyActions = new ArrayList<>();
 932
 933         int order = 0;
 934
 935         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 936         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 937         Action setNextHop;
 938
 939         Set<NodeConnectorId> extPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
 940         if (extPorts == null || !extPorts.iterator().hasNext()) {
 941             LOG.warn("No external interface on node: {}. External Gateway {} is not reachable!", nodeId, l2Gw.getKey());
 942             return null;
 943         }
 944         // only one external port is supported for now
 945          NodeConnectorId extPort = extPorts.iterator().next();
 946
 947         long portNum;
 948         try {
 949             portNum = getOfPortNum(extPort);
 950         } catch (NumberFormatException ex) {
 951             LOG.warn("Could not parse port number {}", extPort, ex);
 952             return null;
 953         }
 954
 955         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 956         // END L3 LOCAL
 957
 958
 959         Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
 960         l3ApplyActions.add(setDlSrc);
 961
 962         Action setDlDst = setDlDstAction(l2Gw.getMacAddress());
 963         l3ApplyActions.add(setDlDst);
 964
 965         order += 1;
 966         applyActions.add(setdEPG);
 967         applyActions.add(setdCG);
 968         applyActions.add(setNextHop);
 969
 970         applyActions.addAll(l3ApplyActions);
 971         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 972             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 973             .build();
 974
 975         l3instructions.add(applyActionsIns);
 976         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 977             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 978             .build();
 979         l3instructions.add(gotoTable);
 980         Layer3Match m = null;
 981         Long etherType = null;
 982         String ikey = null;
 983         if (destL3Address.getIpAddress().getIpv4Address() != null) {
 984             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
 985             etherType = IPv4;
 986             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 987         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
 988             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
 989             etherType = IPv6;
 990             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 991         } else {
 992             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
 993             return null;
 994         }
 995
 996         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
 997             .setLayer3Match(m);
 998         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 999         Match match = mb.build();
1000         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "externalL3", match);
1001         FlowBuilder flowb = base().setId(flowid)
1002             .setPriority(Integer.valueOf(132))
1003             .setMatch(match)
1004             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
1005         return flowb.build();
1006     }
1007
1008     private Endpoint getL2EndpointOfSubnetGateway(TenantId tenantId, Subnet subnet) {
1009         if (subnet != null && subnet.getVirtualRouterIp() != null) {
1010             IpAddress gwIpAddress = subnet.getVirtualRouterIp();
1011             Collection<EndpointL3Prefix> prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(tenantId);
1012             if (prefixEps != null) {
1013                 for (EndpointL3Prefix prefixEp : prefixEps) {
1014                     for (EndpointL3Gateways gw : prefixEp.getEndpointL3Gateways()) {
1015                         EndpointL3 l3Ep = ctx.getEndpointManager().getL3Endpoint(gw.getL3Context(), gwIpAddress,
1016                                 prefixEp.getTenant());
1017                         if (l3Ep != null && l3Ep.getL2Context() != null && l3Ep.getMacAddress() != null) {
1018                             return ctx.getEndpointManager().getEndpoint(
1019                                     new EpKey(l3Ep.getL2Context(), l3Ep.getMacAddress()));
1020                         }
1021                     }
1022                 }
1023             }
1024         }
1025         return null;
1026     }
1027
1028     private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds,
1029             EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) {
1030
1031         // TODO Li alagalah - refactor common code but keep simple method
1032
1033         // this endpoint is on a different switch; send to the
1034         // appropriate tunnel
1035
1036         ArrayList<Instruction> instructions = new ArrayList<>();
1037         List<Action> applyActions = new ArrayList<>();
1038
1039         int order = 0;
1040
1041         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
1042         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
1043         Action setNextHop;
1044         String nextHop;
1045
1046         // BEGIN TUNNEL HANDLING
1047         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
1048         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
1049         if (tunDst == null) {
1050             LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep);
1051             return null;
1052         }
1053         if (tunPort == null) {
1054             LOG.warn("Failed to get Tunnel Port for NodeId {} with EP {}", nodeId, ep);
1055             return null;
1056         }
1057
1058         Action tundstAction;
1059
1060         if (tunDst.getIpv4Address() != null) {
1061             nextHop = tunDst.getIpv4Address().getValue();
1062             tundstAction = nxLoadTunIPv4Action(nextHop, false);
1063         } else if (tunDst.getIpv6Address() != null) {
1064             // nextHop = tunDst.getIpv6Address().getValue();
1065             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
1066                     ofc.getNodeId());
1067             return null;
1068         } else {
1069             // this shouldn't happen
1070             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
1071             return null;
1072         }
1073
1074         long portNum;
1075         try {
1076             portNum = getOfPortNum(tunPort);
1077         } catch (NumberFormatException ex) {
1078             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
1079             return null;
1080         }
1081
1082         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1083         applyActions.add(tundstAction);
1084         // END TUNNEL
1085
1086         order += 1;
1087         applyActions.add(setdEPG);
1088         applyActions.add(setdCG);
1089         applyActions.add(setNextHop);
1090         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1091             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1092             .build();
1093         instructions.add(applyActionsIns);
1094
1095         applyActionsIns = new InstructionBuilder().setOrder(order++)
1096             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1097             .build();
1098
1099         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1100             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1101             .build();
1102         instructions.add(gotoTable);
1103
1104         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
1105         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId())));
1106         Match match = mb.build();
1107         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL2", match);
1108         FlowBuilder flowb = base().setId(flowid)
1109             .setPriority(Integer.valueOf(50))
1110             .setMatch(match)
1111             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
1112
1113         return flowb.build();
1114     }
1115
1116     private Flow createExternalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds,NodeId nodeId) {
1117
1118         ArrayList<Instruction> instructions = new ArrayList<>();
1119         List<Action> applyActions = new ArrayList<>();
1120
1121         int order = 0;
1122
1123         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
1124         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
1125         Action setNextHop;
1126
1127         // BEGIN L2 LOCAL
1128         Set<NodeConnectorId> extPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
1129         if(extPorts == null || !extPorts.iterator().hasNext()) {
1130             return null;
1131         }
1132         // Only one external port is currently supported.
1133         NodeConnectorId extPort = extPorts.iterator().next();
1134         long portNum;
1135         try {
1136             portNum = getOfPortNum(extPort);
1137         } catch (NumberFormatException ex) {
1138             LOG.warn("Could not parse port number {}", extPort, ex);
1139             return null;
1140         }
1141         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1142
1143         // END L2 LOCAL
1144
1145         order += 1;
1146         applyActions.add(setdEPG);
1147         applyActions.add(setdCG);
1148         applyActions.add(setNextHop);
1149         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1150             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1151             .build();
1152         instructions.add(applyActionsIns);
1153
1154         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1155             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1156             .build();
1157         instructions.add(gotoTable);
1158
1159         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
1160         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
1161         Match match = mb.build();
1162         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "externalL2", match);
1163         FlowBuilder flowb = base().setId(flowid)
1164             .setPriority(Integer.valueOf(50))
1165             .setMatch(match)
1166             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
1167         return flowb.build();
1168     }
1169
1170     private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId,
1171             EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc,
1172             Subnet srcSubnet) {
1173
1174         // TODO Li alagalah - refactor common code but keep simple method
1175
1176         // this endpoint is on a different switch; send to the
1177         // appropriate tunnel
1178         Subnet destSubnet = null;
1179         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
1180         if (subnets == null) {
1181             LOG.trace("No subnets in tenant {}", destEp.getTenant());
1182             return null;
1183         }
1184         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
1185         for (Subnet subnet : subnets) {
1186             // TODO Li alagalah add IPv6 support
1187             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
1188                 destSubnet = subnet;
1189                 break;
1190             }
1191         }
1192         if (destSubnet == null) {
1193             LOG.info("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
1194             return null;
1195         }
1196
1197         if (destSubnet.getVirtualRouterIp() == null) {
1198             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
1199                     destL3Address.getKey());
1200             return null;
1201         }
1202
1203         if (srcSubnet.getVirtualRouterIp() == null) {
1204             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
1205             return null;
1206         }
1207         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
1208         if (destL3c == null || destL3c.getId() == null) {
1209             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
1210             return null;
1211         }
1212         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
1213         if (srcL3c == null || srcL3c.getId() == null) {
1214             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
1215             return null;
1216         }
1217
1218         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
1219             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
1220                 .getValue(), destL3c.getId().getValue());
1221             return null;
1222         }
1223
1224         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
1225         MacAddress epDestMac = destEp.getMacAddress();
1226         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
1227         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
1228             // This is our final destination, so match on actual EP mac.
1229             matcherMac = epDestMac;
1230         }
1231         ArrayList<Instruction> l3instructions = new ArrayList<>();
1232         List<Action> applyActions = new ArrayList<>();
1233         List<Action> l3ApplyActions = new ArrayList<>();
1234
1235         int order = 0;
1236
1237         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
1238         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
1239         Action setNextHop;
1240         String nextHop;
1241
1242         // BEGIN TUNNEL HANDLING
1243         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
1244         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
1245         if (tunDst == null) {
1246             LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address);
1247             return null;
1248         }
1249         if (tunPort == null) {
1250             LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, destL3Address);
1251             return null;
1252         }
1253
1254         Action tundstAction;
1255
1256         if (tunDst.getIpv4Address() != null) {
1257             nextHop = tunDst.getIpv4Address().getValue();
1258             tundstAction = nxLoadTunIPv4Action(nextHop, false);
1259         } else if (tunDst.getIpv6Address() != null) {
1260             // nextHop = tunDst.getIpv6Address().getValue();
1261             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
1262                     ofc.getNodeId());
1263             return null;
1264         } else {
1265             // this shouldn't happen
1266             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
1267             return null;
1268         }
1269
1270         long portNum;
1271         try {
1272             portNum = getOfPortNum(tunPort);
1273         } catch (NumberFormatException ex) {
1274             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
1275             return null;
1276         }
1277
1278         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1279         applyActions.add(tundstAction);
1280         // END TUNNEL
1281
1282         order += 1;
1283         applyActions.add(setdEPG);
1284         applyActions.add(setdCG);
1285         applyActions.add(setNextHop);
1286
1287         // Lets not re-write the srcMac if its local.
1288         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
1289             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
1290             l3ApplyActions.add(setDlSrc);
1291         }
1292
1293         Action setDlDst = setDlDstAction(epDestMac);
1294         l3ApplyActions.add(setDlDst);
1295
1296         Action decTtl = decNwTtlAction();
1297         l3ApplyActions.add(decTtl);
1298
1299         applyActions.addAll(l3ApplyActions);
1300         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1301             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1302             .build();
1303
1304         l3instructions.add(applyActionsIns);
1305         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1306             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1307             .build();
1308         l3instructions.add(gotoTable);
1309         Layer3Match m = null;
1310         Long etherType = null;
1311         String ikey = null;
1312         if (destL3Address.getIpAddress().getIpv4Address() != null) {
1313             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
1314             etherType = IPv4;
1315             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
1316         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
1317             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
1318             etherType = IPv6;
1319             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
1320         } else {
1321             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
1322             return null;
1323         }
1324
1325         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
1326             .setLayer3Match(m);
1327         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id())));
1328         Match match = mb.build();
1329         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL3", match);
1330         FlowBuilder flowb = base().setId(flowid)
1331             .setPriority(Integer.valueOf(132))
1332             .setMatch(match)
1333             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
1334         return flowb.build();
1335     }
1336
1337     private NetworkDomainId getEPNetworkContainment(Endpoint endpoint) {
1338         if (endpoint.getNetworkContainment() != null) {
1339             return endpoint.getNetworkContainment();
1340         } else {
1341             /*
1342              * TODO: Be alagalah: Endpoint Refactor: This should be set on input
1343              * which we can't do because of the backwards way endpoints were
1344              * "architected".
1345              */
1346             return ctx.getTenant(endpoint.getTenant())
1347                 .getEndpointGroup(endpoint.getEndpointGroup())
1348                 .getNetworkDomain();
1349         }
1350     }
1351
1352     private HashSet<Subnet> getSubnets(final TenantId tenantId) {
1353
1354         if (ctx.getDataBroker() == null) {
1355             return null;
1356         }
1357
1358         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
1359         InstanceIdentifier<Tenant> tiid = TenantUtils.tenantIid(tenantId);
1360         Optional<Tenant> tenantInfo;
1361         try {
1362             tenantInfo = t.read(LogicalDatastoreType.CONFIGURATION, tiid).get();
1363         } catch (Exception e) {
1364             LOG.error("Could not read Tenant {}", tenantId, e);
1365             return null;
1366         } finally {
1367             t.close();
1368         }
1369
1370         if (!tenantInfo.isPresent()) {
1371             LOG.warn("Tenant {} not found", tenantId);
1372             return null;
1373         }
1374
1375         ForwardingContext fwCtx = tenantInfo.get().getForwardingContext();
1376         if (fwCtx == null || fwCtx.getSubnet() == null) {
1377             return new HashSet<>();
1378         }
1379         return new HashSet<>(fwCtx.getSubnet());
1380     }
1381
1382     // Need a method to get subnets for EPs attached to the node locally
1383     // to set the source Mac address for the router interface.
1384     private List<Subnet> getLocalSubnets(NodeId nodeId) {
1385         Collection<Endpoint> endpointsForNode = ctx.getEndpointManager().getEndpointsForNode(nodeId);
1386
1387         List<Subnet> localSubnets = new ArrayList<Subnet>();
1388
1389         for (Endpoint endpoint : endpointsForNode) {
1390             HashSet<Subnet> subnets = getSubnets(endpoint.getTenant());
1391             if (subnets == null) {
1392                 LOG.debug("No local subnets in tenant {} for EP {}.", endpoint.getTenant(), endpoint.getKey());
1393                 continue;
1394             }
1395             NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint);
1396             for (Subnet subnet : subnets) {
1397                 if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) {
1398                     localSubnets.add(subnet);
1399                 }
1400             }
1401         }
1402         return localSubnets;
1403     }
1404
1405     static byte[] bytesFromHexString(String values) {
1406         String target = "";
1407         if (values != null) {
1408             target = values;
1409         }
1410         String[] octets = target.split(":");
1411
1412         byte[] ret = new byte[octets.length];
1413         for (int i = 0; i < octets.length; i++) {
1414             ret[i] = Integer.valueOf(octets[i], 16).byteValue();
1415         }
1416         return ret;
1417     }
1418 }