renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java

   1 /*
   2  * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow;
  10
  11 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP;
  12 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4;
  13 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6;
  14 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch;
  15 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns;
  16 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.createNodePath;
  17 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction;
  18 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch;
  19 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum;
  20 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.gotoTableIns;
  21 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.groupAction;
  22 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.instructions;
  23 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpOpAction;
  24 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpShaAction;
  25 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction;
  26 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction;
  27 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action;
  28 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction;
  29 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction;
  30 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction;
  31 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction;
  32 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction;
  33 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction;
  34 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction;
  35
  36 import java.math.BigInteger;
  37 import java.util.ArrayList;
  38 import java.util.Collection;
  39 import java.util.Collections;
  40 import java.util.HashMap;
  41 import java.util.HashSet;
  42 import java.util.List;
  43 import java.util.Map;
  44 import java.util.Map.Entry;
  45 import java.util.Objects;
  46 import java.util.Set;
  47
  48 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  49 import org.opendaylight.controller.md.sal.binding.api.ReadTransaction;
  50 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  51 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  52 import org.opendaylight.groupbasedpolicy.endpoint.EpKey;
  53 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext;
  54 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.PolicyManager.FlowMap;
  55 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch;
  56 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals;
  57 import org.opendaylight.groupbasedpolicy.resolver.EgKey;
  58 import org.opendaylight.groupbasedpolicy.resolver.PolicyInfo;
  59 import org.opendaylight.groupbasedpolicy.resolver.TenantUtils;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix;
  63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.EndpointLocation.LocationType;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.L3Context;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.Subnet;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.ethernet.match.fields.EthernetDestinationBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatchBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.Layer3Match;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 103 import org.opendaylight.yangtools.yang.binding.DataObject;
 104 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 105 import org.slf4j.Logger;
 106 import org.slf4j.LoggerFactory;
 107
 108 import com.google.common.base.Optional;
 109 import com.google.common.collect.HashMultimap;
 110 import com.google.common.collect.SetMultimap;
 111 import com.google.common.collect.Sets;
 112 import com.google.common.util.concurrent.CheckedFuture;
 113
 114 /**
 115  * Manage the table that maps the destination address to the next hop for the
 116  * path as well as applies any relevant routing transformations.
 117  */
 118 public class DestinationMapper extends FlowTable {
 119
 120     protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class);
 121
 122     // TODO Li alagalah: Improve UT coverage for this class.
 123
 124     // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows
 125     // This ensures we have the appropriate network-containment'
 126
 127     public static final short TABLE_ID = 2;
 128     /**
 129      * This is the MAC address of the magical router in the sky
 130      */
 131     public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5");
 132     public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00");
 133
 134     public DestinationMapper(OfContext ctx) {
 135         super(ctx);
 136     }
 137
 138     Map<TenantId, HashSet<Subnet>> subnetsByTenant = new HashMap<TenantId, HashSet<Subnet>>();
 139
 140     @Override
 141     public short getTableId() {
 142         return TABLE_ID;
 143     }
 144
 145     @Override
 146     public void sync(NodeId nodeId, PolicyInfo policyInfo, FlowMap flowMap) throws Exception {
 147
 148         TenantId currentTenant;
 149
 150         flowMap.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null));
 151
 152         SetMultimap<EpKey, EpKey> visitedEps = HashMultimap.create();
 153         Set<EndpointFwdCtxOrdinals> epOrdSet = new HashSet<>();
 154
 155         for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) {
 156             Set<EndpointGroupId> srcEpgIds = new HashSet<>();
 157             if (srcEp.getEndpointGroup() != null)
 158                 srcEpgIds.add(srcEp.getEndpointGroup());
 159             if (srcEp.getEndpointGroups() != null)
 160                 srcEpgIds.addAll(srcEp.getEndpointGroups());
 161
 162             for (EndpointGroupId epgId : srcEpgIds) {
 163                 EgKey epg = new EgKey(srcEp.getTenant(), epgId);
 164                 Set<EgKey> peers = Sets.union(Collections.singleton(epg), policyInfo.getPeers(epg));
 165                 for (EgKey peer : peers) {
 166                     for (Endpoint peerEp : ctx.getEndpointManager().getEndpointsForGroup(peer)) {
 167                         currentTenant = peerEp.getTenant();
 168                         subnetsByTenant.put(currentTenant, getSubnets(currentTenant));
 169                         EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress());
 170                         EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress());
 171
 172                         if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) {
 173                             continue;
 174                         }
 175                         syncEP(flowMap, nodeId, policyInfo, srcEp, peerEp);
 176                         visitedEps.put(srcEpKey, peerEpKey);
 177
 178                         // Process subnets and flood-domains for epPeer
 179                         EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo,
 180                                 peerEp);
 181                         epOrdSet.add(epOrds);
 182                     }
 183                 }
 184             }
 185         }
 186
 187         for (Entry<TenantId, HashSet<Subnet>> subnetEntry : subnetsByTenant.entrySet()) {
 188             if (subnetEntry.getValue() == null) {
 189                 LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey());
 190                 continue;
 191             }
 192             currentTenant = subnetEntry.getKey();
 193             for (Subnet sn : subnetEntry.getValue()) {
 194                 L3Context l3c = getL3ContextForSubnet(currentTenant, sn);
 195                 Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn,
 196                         OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId()));
 197                 if (arpFlow != null) {
 198                     flowMap.writeFlow(nodeId, TABLE_ID, arpFlow);
 199                 } else {
 200                     LOG.debug(
 201                             "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .",
 202                             sn.getIpPrefix().getValue());
 203                 }
 204             }
 205         }
 206
 207         // Write broadcast flows per flood domain.
 208         for (EndpointFwdCtxOrdinals epOrd : epOrdSet) {
 209             if (groupExists(nodeId, epOrd.getFdId())) {
 210                 flowMap.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd));
 211             }
 212         }
 213     }
 214
 215     // set up next-hop destinations for all the endpoints in the endpoint
 216     // group on the node
 217
 218     private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) {
 219         FlowId flowId = new FlowId(new StringBuilder().
 220                                     append("broadcast|").
 221                                     append(epOrd.getFdId()).
 222                                     toString());
 223         MatchBuilder mb = new MatchBuilder()
 224                             .setEthernetMatch(new EthernetMatchBuilder()
 225                             .setEthernetDestination(new EthernetDestinationBuilder().
 226                                                         setAddress(MULTICAST_MAC)
 227                                                         .setMask(MULTICAST_MAC).build())
 228                             .build());
 229         addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId())));
 230
 231         FlowBuilder flowb = base().setPriority(Integer.valueOf(140))
 232             .setId(flowId)
 233             .setMatch(mb.build())
 234             .setInstructions(
 235                     instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false),
 236                             groupAction(Long.valueOf(epOrd.getFdId())))));
 237
 238         return flowb.build();
 239     }
 240
 241     private boolean groupExists(NodeId nodeId, Integer fdId) throws Exception {
 242         // Fetch existing GroupTables
 243         if (ctx.getDataBroker() == null) {
 244             return false;
 245         }
 246
 247         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 248         InstanceIdentifier<Node> niid = createNodePath(nodeId);
 249         Optional<Node> r = t.read(LogicalDatastoreType.CONFIGURATION, niid).get();
 250         if (!r.isPresent())
 251             return false;
 252         FlowCapableNode fcn = r.get().getAugmentation(FlowCapableNode.class);
 253         if (fcn == null)
 254             return false;
 255
 256         if (fcn.getGroup() != null) {
 257             for (Group g : fcn.getGroup()) {
 258                 if (g.getGroupId().getValue().equals(Long.valueOf(fdId))) { // Group
 259                                                                             // Exists.
 260                     return true;
 261                 }
 262             }
 263         }
 264         return false;
 265     }
 266
 267     private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) {
 268
 269         if (ctx.getDataBroker() == null) {
 270             return null;
 271         }
 272
 273         MacAddress defaultMacAddress = ROUTER_MAC;
 274
 275         EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId());
 276         InstanceIdentifier<EndpointL3> endpointsIid = InstanceIdentifier.builder(Endpoints.class)
 277             .child(EndpointL3.class, l3Key)
 278             .build();
 279         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 280
 281         Optional<EndpointL3> r;
 282         try {
 283             r = t.read(LogicalDatastoreType.OPERATIONAL, endpointsIid).get();
 284             if (!r.isPresent())
 285                 return defaultMacAddress;
 286             EndpointL3 epL3 = r.get();
 287             if (epL3.getMacAddress() == null) {
 288                 return defaultMacAddress;
 289             } else {
 290                 return epL3.getMacAddress();
 291             }
 292         } catch (Exception e) {
 293             LOG.error("Error reading EndpointL3 {}.{}", l3c, ipAddress, e);
 294             return null;
 295         }
 296     }
 297
 298     private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) {
 299         L3Context l3c = ctx.getPolicyResolver().getTenant(tenantId).resolveL3Context(sn.getId());
 300         return l3c;
 301     }
 302
 303     private Flow createRouterArpFlow(TenantId tenantId, NodeId nodeId, Subnet sn, int l3Id) {
 304         if (sn == null || sn.getVirtualRouterIp() == null) {
 305             LOG.trace("Didn't create routerArpFlow since either subnet or subnet virtual router was null");
 306             return null;
 307         }
 308         /*
 309          * TODO: Li alagalah: This should be new Yang "gateways" list as well,
 310          * that expresses the gateway and prefixes it is interface for. Should
 311          * also check for external.
 312          */
 313         if (sn.getVirtualRouterIp().getIpv4Address() != null) {
 314             String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue();
 315
 316             L3Context l3c = getL3ContextForSubnet(tenantId, sn);
 317             if (l3c == null) {
 318                 LOG.error("No L3 Context found associated with subnet {}", sn.getId());
 319             }
 320
 321             MacAddress routerMac = routerPortMac(l3c, sn.getVirtualRouterIp());
 322             if (routerMac == null) {
 323                 return null;
 324             }
 325
 326             BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue()));
 327
 328             FlowId flowId = new FlowId(new StringBuffer().append("routerarp|")
 329                 .append(sn.getId().getValue())
 330                 .append("|")
 331                 .append(ikey)
 332                 .append("|")
 333                 .append(l3Id)
 334                 .toString());
 335             MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match(
 336                     new ArpMatchBuilder().setArpOp(Integer.valueOf(1))
 337                         .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32"))
 338                         .build());
 339             addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id)));
 340
 341             FlowBuilder flowb = base().setPriority(150)
 342                 .setId(flowId)
 343                 .setMatch(mb.build())
 344                 .setInstructions(
 345                         instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac),
 346                                 nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(),
 347                                 nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(),
 348                                 nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue()
 349                                         + ":INPORT")))));
 350             return flowb.build();
 351         } else {
 352             LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId()
 353                 .getValue());
 354             return null;
 355         }
 356
 357     }
 358
 359     private Flow createLocalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) {
 360
 361         // TODO Li alagalah - refactor common code but keep simple method
 362         ArrayList<Instruction> instructions = new ArrayList<>();
 363         List<Action> applyActions = new ArrayList<>();
 364
 365         int order = 0;
 366
 367         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 368         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 369         Action setNextHop;
 370         String nextHop;
 371
 372         // BEGIN L2 LOCAL
 373         nextHop = ofc.getNodeConnectorId().getValue();
 374
 375         long portNum;
 376         try {
 377             portNum = getOfPortNum(ofc.getNodeConnectorId());
 378         } catch (NumberFormatException ex) {
 379             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 380             return null;
 381         }
 382
 383         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 384
 385         // END L2 LOCAL
 386
 387         order += 1;
 388         applyActions.add(setdEPG);
 389         applyActions.add(setdCG);
 390         applyActions.add(setNextHop);
 391         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 392             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 393             .build();
 394         instructions.add(applyActionsIns);
 395
 396         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 397             .setInstruction(gotoTableIns((short) (getTableId() + 1)))
 398             .build();
 399         instructions.add(gotoTable);
 400
 401         FlowId flowid = new FlowId(new StringBuilder().append(epFwdCtxOrds.getBdId())
 402             .append("|l2|")
 403             .append(ep.getMacAddress().getValue())
 404             .append("|")
 405             .append(nextHop)
 406             .toString());
 407         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 408         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
 409         FlowBuilder flowb = base().setId(flowid)
 410             .setPriority(Integer.valueOf(50))
 411             .setMatch(mb.build())
 412             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 413         return flowb.build();
 414     }
 415
 416     private void syncEP(FlowMap flowMap, NodeId nodeId, PolicyInfo policyInfo, Endpoint srcEp, Endpoint destEp)
 417             throws Exception {
 418
 419         // TODO: Conditions messed up, but for now, send policyInfo until this
 420         // is fixed.
 421         EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, destEp);
 422         EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, srcEp);
 423
 424         if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) {
 425             LOG.trace("Didn't process endpoint due to either tenant, or EPG(s) being null", destEp.getKey());
 426             return;
 427         }
 428         OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class);
 429
 430         // ////////////////////////////////////////////////////////////////////////////////////////
 431         /*
 432          * NOT HANDLING EXTERNALS TODO: alagalah Li: External Gateway
 433          * functionality needed here.
 434          */
 435         if (LocationType.External.equals(ofc.getLocationType())) {
 436             // XXX - TODO - perform NAT and send to the external network
 437             // TODO: Use case neutron gateway interface
 438             LOG.warn("External endpoints not yet supported");
 439             return;
 440         }
 441
 442         /*
 443          * Only care about subnets for L3, but fetch them before loop. We need
 444          * the local subnets for setting SRC MAC for routing. All Routing is now
 445          * done locally! YAY! Instead of being shovelled L2 style across network
 446          * ala Helium.
 447          */
 448         List<Subnet> localSubnets = getLocalSubnets(nodeId);
 449         if (localSubnets == null) {
 450             LOG.error("No subnets could be found locally for node: {}", nodeId);
 451             return;
 452         }
 453
 454         if (Objects.equals(ofc.getNodeId(), nodeId)) {
 455             // this is a local endpoint; send to the approppriate local
 456             // port
 457
 458             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 459                 flowMap.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc));
 460             }
 461             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 462             // The Endpoint conflation must end!
 463             if (destEp.getL3Address() == null) {
 464                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 465                 return;
 466             }
 467
 468             for (L3Address l3a : destEp.getL3Address()) {
 469                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 470                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 471                             destEp.getL3Address());
 472                     continue;
 473                 } else {
 474                     for (Subnet localSubnet : localSubnets) {
 475                         Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet);
 476                         if (flow != null) {
 477                             flowMap.writeFlow(nodeId, TABLE_ID, flow);
 478                         } else {
 479                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 480                                     localSubnet.getIpPrefix().getValue());
 481                         }
 482                     }
 483                 }
 484             }
 485         } else {
 486             // this endpoint is on a different switch; send to the
 487             // appropriate tunnel
 488             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 489                 Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc);
 490                 if (remoteL2Flow != null) {
 491                     flowMap.writeFlow(nodeId, TABLE_ID, remoteL2Flow);
 492                 }
 493             } else {
 494                 LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}",
 495                         srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId());
 496             }
 497
 498             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 499             // The Endpoint conflation must end!
 500             if (destEp.getL3Address() == null) {
 501                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 502                 return;
 503             }
 504             for (L3Address l3a : destEp.getL3Address()) {
 505                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 506                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 507                             destEp.getL3Address());
 508                     continue;
 509                 } else {
 510                     for (Subnet localSubnet : localSubnets) {
 511                         Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds,
 512                                 destEpFwdCtxOrds, ofc, localSubnet);
 513                         if (remoteL3Flow != null) {
 514                             flowMap.writeFlow(nodeId, TABLE_ID, remoteL3Flow);
 515                         } else {
 516                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 517                                     localSubnet.getIpPrefix().getValue());
 518                         }
 519                     }
 520                 }
 521             }
 522         } // remote (tunnel)
 523
 524         // }
 525
 526     }
 527
 528     /*
 529      * ################################## DestMapper Flow methods
 530      * ##################################
 531      */
 532     private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
 533             OfOverlayContext ofc, Subnet srcSubnet) {
 534
 535         // TODO Li alagalah - refactor common code but keep simple method
 536
 537         Subnet destSubnet = null;
 538         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 539         if (subnets == null) {
 540             LOG.trace("No subnets in tenant {}", destL3Address.getIpAddress());
 541             return null;
 542         }
 543         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 544         for (Subnet subnet : subnets) {
 545             // TODO Li alagalah add IPv6 support
 546             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 547                 destSubnet = subnet;
 548                 break;
 549             }
 550         }
 551         if (destSubnet == null) {
 552             LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 553             return null;
 554         }
 555
 556         if (destSubnet.getVirtualRouterIp() == null) {
 557             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 558                     destL3Address.getKey());
 559             return null;
 560         }
 561
 562         if (srcSubnet.getVirtualRouterIp() == null) {
 563             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 564             return null;
 565         }
 566         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 567         if (destL3c == null || destL3c.getId() == null) {
 568             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 569             return null;
 570         }
 571         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 572         if (srcL3c == null || srcL3c.getId() == null) {
 573             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 574             return null;
 575         }
 576
 577         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 578             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 579                 .getValue(), destL3c.getId().getValue());
 580             return null;
 581         }
 582
 583         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 584         MacAddress epDestMac = destEp.getMacAddress();
 585         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 586
 587         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 588             // This is our final destination, so match on actual EP mac.
 589             matcherMac = epDestMac;
 590         }
 591
 592         ArrayList<Instruction> l3instructions = new ArrayList<>();
 593         List<Action> applyActions = new ArrayList<>();
 594         List<Action> l3ApplyActions = new ArrayList<>();
 595
 596         int order = 0;
 597
 598         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 599         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 600         Action setNextHop;
 601         String nextHop;
 602
 603         // BEGIN L3 LOCAL
 604         nextHop = ofc.getNodeConnectorId().getValue();
 605
 606         long portNum;
 607         try {
 608             portNum = getOfPortNum(ofc.getNodeConnectorId());
 609         } catch (NumberFormatException ex) {
 610             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 611             return null;
 612         }
 613
 614         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 615         // END L3 LOCAL
 616
 617         // Lets not re-write the srcMac if its local.
 618         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
 619             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
 620             l3ApplyActions.add(setDlSrc);
 621         }
 622
 623         Action setDlDst = setDlDstAction(epDestMac);
 624         l3ApplyActions.add(setDlDst);
 625
 626         Action decTtl = decNwTtlAction();
 627         l3ApplyActions.add(decTtl);
 628
 629         order += 1;
 630         applyActions.add(setdEPG);
 631         applyActions.add(setdCG);
 632         applyActions.add(setNextHop);
 633
 634         applyActions.addAll(l3ApplyActions);
 635         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 636             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 637             .build();
 638
 639         l3instructions.add(applyActionsIns);
 640         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 641             .setInstruction(gotoTableIns((short) (getTableId() + 1)))
 642             .build();
 643         l3instructions.add(gotoTable);
 644         Layer3Match m = null;
 645         Long etherType = null;
 646         String ikey = null;
 647         if (destL3Address.getIpAddress().getIpv4Address() != null) {
 648             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
 649             etherType = IPv4;
 650             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 651         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
 652             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
 653             etherType = IPv6;
 654             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 655         } else {
 656             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
 657             return null;
 658         }
 659
 660         FlowId flowid = new FlowId(new StringBuilder().append(Integer.toString(epFwdCtxOrds.getL3Id()))
 661             .append("|l3|")
 662             .append(ikey)
 663             .append("|")
 664             .append(Integer.toString(epFwdCtxOrds.getEpgId()))
 665             .append("|")
 666             .append(Integer.toString(epFwdCtxOrds.getCgId()))
 667             .append("|")
 668             .append(matcherMac)
 669             .append("|")
 670             .append(destSubnetGatewayMac)
 671             .append("|")
 672             .append(nextHop)
 673             .toString());
 674         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
 675             .setLayer3Match(m);
 676         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 677         FlowBuilder flowb = base().setId(flowid)
 678             .setPriority(Integer.valueOf(132))
 679             .setMatch(mb.build())
 680             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 681         return flowb.build();
 682     }
 683
 684     private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds,
 685             EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) {
 686
 687         // TODO Li alagalah - refactor common code but keep simple method
 688
 689         // this endpoint is on a different switch; send to the
 690         // appropriate tunnel
 691
 692         ArrayList<Instruction> instructions = new ArrayList<>();
 693         List<Action> applyActions = new ArrayList<>();
 694
 695         int order = 0;
 696
 697         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
 698         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
 699         Action setNextHop;
 700         String nextHop;
 701
 702         // BEGIN TUNNEL HANDLING
 703         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
 704         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
 705         if (tunDst == null) {
 706             LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep);
 707             return null;
 708         }
 709         if (tunPort == null) {
 710             LOG.warn("Failed to get Tunnel Port for NodeId {} with EP {}", nodeId, ep);
 711             return null;
 712         }
 713
 714         Action tundstAction;
 715
 716         if (tunDst.getIpv4Address() != null) {
 717             nextHop = tunDst.getIpv4Address().getValue();
 718             tundstAction = nxLoadTunIPv4Action(nextHop, false);
 719         } else if (tunDst.getIpv6Address() != null) {
 720             // nextHop = tunDst.getIpv6Address().getValue();
 721             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
 722                     ofc.getNodeId());
 723             return null;
 724         } else {
 725             // this shouldn't happen
 726             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
 727             return null;
 728         }
 729
 730         long portNum;
 731         try {
 732             portNum = getOfPortNum(tunPort);
 733         } catch (NumberFormatException ex) {
 734             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 735             return null;
 736         }
 737
 738         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 739         applyActions.add(tundstAction);
 740         // END TUNNEL
 741
 742         order += 1;
 743         applyActions.add(setdEPG);
 744         applyActions.add(setdCG);
 745         applyActions.add(setNextHop);
 746         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 747             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 748             .build();
 749         instructions.add(applyActionsIns);
 750
 751         applyActionsIns = new InstructionBuilder().setOrder(order++)
 752             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 753             .build();
 754
 755         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 756             .setInstruction(gotoTableIns((short) (getTableId() + 1)))
 757             .build();
 758         instructions.add(gotoTable);
 759
 760         FlowId flowid = new FlowId(new StringBuilder().append(destEpFwdCtxOrds.getBdId())
 761             .append("|l2|")
 762             .append(ep.getMacAddress().getValue())
 763             .append("|")
 764             .append(srcEpFwdCtxOrds.getBdId())
 765             .append("|")
 766             .append(nextHop)
 767             .toString());
 768         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 769         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId())));
 770         FlowBuilder flowb = base().setId(flowid)
 771             .setPriority(Integer.valueOf(50))
 772             .setMatch(mb.build())
 773             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 774
 775         return flowb.build();
 776     }
 777
 778     private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId,
 779             EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc,
 780             Subnet srcSubnet) {
 781
 782         // TODO Li alagalah - refactor common code but keep simple method
 783
 784         // this endpoint is on a different switch; send to the
 785         // appropriate tunnel
 786         Subnet destSubnet = null;
 787         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 788         if (subnets == null) {
 789             LOG.trace("No subnets in tenant {}", destL3Address.getIpAddress());
 790             return null;
 791         }
 792         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 793         for (Subnet subnet : subnets) {
 794             // TODO Li alagalah add IPv6 support
 795             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 796                 destSubnet = subnet;
 797                 break;
 798             }
 799         }
 800         if (destSubnet == null) {
 801             LOG.info("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 802             return null;
 803         }
 804
 805         if (destSubnet.getVirtualRouterIp() == null) {
 806             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 807                     destL3Address.getKey());
 808             return null;
 809         }
 810
 811         if (srcSubnet.getVirtualRouterIp() == null) {
 812             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 813             return null;
 814         }
 815         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 816         if (destL3c == null || destL3c.getId() == null) {
 817             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 818             return null;
 819         }
 820         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 821         if (srcL3c == null || srcL3c.getId() == null) {
 822             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 823             return null;
 824         }
 825
 826         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 827             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 828                 .getValue(), destL3c.getId().getValue());
 829             return null;
 830         }
 831
 832         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 833         MacAddress epDestMac = destEp.getMacAddress();
 834         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 835
 836         ArrayList<Instruction> l3instructions = new ArrayList<>();
 837         List<Action> applyActions = new ArrayList<>();
 838         List<Action> l3ApplyActions = new ArrayList<>();
 839
 840         int order = 0;
 841
 842         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
 843         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
 844         Action setNextHop;
 845         String nextHop;
 846
 847         // BEGIN TUNNEL HANDLING
 848         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
 849         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
 850         if (tunDst == null) {
 851             LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address);
 852             return null;
 853         }
 854         if (tunPort == null) {
 855             LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, destL3Address);
 856             return null;
 857         }
 858
 859         Action tundstAction;
 860
 861         if (tunDst.getIpv4Address() != null) {
 862             nextHop = tunDst.getIpv4Address().getValue();
 863             tundstAction = nxLoadTunIPv4Action(nextHop, false);
 864         } else if (tunDst.getIpv6Address() != null) {
 865             // nextHop = tunDst.getIpv6Address().getValue();
 866             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
 867                     ofc.getNodeId());
 868             return null;
 869         } else {
 870             // this shouldn't happen
 871             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
 872             return null;
 873         }
 874
 875         long portNum;
 876         try {
 877             portNum = getOfPortNum(tunPort);
 878         } catch (NumberFormatException ex) {
 879             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 880             return null;
 881         }
 882
 883         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 884         applyActions.add(tundstAction);
 885         // END TUNNEL
 886
 887         order += 1;
 888         applyActions.add(setdEPG);
 889         applyActions.add(setdCG);
 890         applyActions.add(setNextHop);
 891
 892         Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
 893         l3ApplyActions.add(setDlSrc);
 894
 895         Action setDlDst = setDlDstAction(epDestMac);
 896         l3ApplyActions.add(setDlDst);
 897
 898         Action decTtl = decNwTtlAction();
 899         l3ApplyActions.add(decTtl);
 900
 901         applyActions.addAll(l3ApplyActions);
 902         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 903             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 904             .build();
 905
 906         l3instructions.add(applyActionsIns);
 907         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 908             .setInstruction(gotoTableIns((short) (getTableId() + 1)))
 909             .build();
 910         l3instructions.add(gotoTable);
 911         Layer3Match m = null;
 912         Long etherType = null;
 913         String ikey = null;
 914         if (destL3Address.getIpAddress().getIpv4Address() != null) {
 915             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
 916             etherType = IPv4;
 917             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 918         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
 919             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
 920             etherType = IPv6;
 921             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 922         } else {
 923             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
 924             return null;
 925         }
 926
 927         FlowId flowid = new FlowId(new StringBuilder().append(Integer.toString(destEpFwdCtxOrds.getL3Id()))
 928             .append("|l3|")
 929             .append(ikey)
 930             .append("|")
 931             .append(matcherMac)
 932             .append("|")
 933             .append(destSubnetGatewayMac)
 934             .append("|")
 935             .append(srcEpFwdCtxOrds.getL3Id())
 936             .append("|")
 937             .append(nextHop)
 938             .toString());
 939         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
 940             .setLayer3Match(m);
 941         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id())));
 942
 943         FlowBuilder flowb = base().setId(flowid)
 944             .setPriority(Integer.valueOf(132))
 945             .setMatch(mb.build())
 946             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 947         return flowb.build();
 948     }
 949
 950     private NetworkDomainId getEPNetworkContainment(Endpoint endpoint) {
 951         if (endpoint.getNetworkContainment() != null) {
 952             return endpoint.getNetworkContainment();
 953         } else {
 954             /*
 955              * TODO: Be alagalah: Endpoint Refactor: This should be set on input
 956              * which we can't do because of the backwards way endpoints were
 957              * "architected".
 958              */
 959             return ctx.getPolicyResolver()
 960                 .getTenant(endpoint.getTenant())
 961                 .getEndpointGroup(endpoint.getEndpointGroup())
 962                 .getNetworkDomain();
 963         }
 964     }
 965
 966     private HashSet<Subnet> getSubnets(final TenantId tenantId) {
 967
 968         // if (subnetsByTenant.get(tenantId) != null) {
 969         // return subnetsByTenant.get(tenantId);
 970         // }
 971
 972         if (ctx.getDataBroker() == null) {
 973             return null;
 974         }
 975
 976         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 977         InstanceIdentifier<Tenant> tiid = TenantUtils.tenantIid(tenantId);
 978         Optional<Tenant> tenantInfo;
 979         try {
 980             tenantInfo = t.read(LogicalDatastoreType.CONFIGURATION, tiid).get();
 981         } catch (Exception e) {
 982             LOG.error("Could not read Tenant {}", tenantId, e);
 983             return null;
 984         }
 985
 986         HashSet<Subnet> subnets = new HashSet<Subnet>();
 987
 988         if (!tenantInfo.isPresent()) {
 989             LOG.warn("Tenant {} not found", tenantId);
 990             return null;
 991         }
 992
 993         subnets.addAll(tenantInfo.get().getSubnet());
 994         // subnetsByTenant.put(tenantId, subnets);
 995         return subnets;
 996     }
 997
 998     // Need a method to get subnets for EPs attached to the node locally
 999     // to set the source Mac address for the router interface.
1000     private List<Subnet> getLocalSubnets(NodeId nodeId) {
1001         Collection<Endpoint> endpointsForNode = ctx.getEndpointManager().getEndpointsForNode(nodeId);
1002
1003         List<Subnet> localSubnets = new ArrayList<Subnet>();
1004
1005         for (Endpoint endpoint : endpointsForNode) {
1006             HashSet<Subnet> subnets = getSubnets(endpoint.getTenant());
1007             if (subnets == null) {
1008                 LOG.error("No local subnets.");
1009                 return null;
1010             }
1011             NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint);
1012             for (Subnet subnet : subnets) {
1013                 if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) {
1014                     localSubnets.add(subnet);
1015                 }
1016             }
1017         }
1018         return localSubnets;
1019     }
1020
1021     /**
1022      * Reads data from datastore as synchronous call.
1023      *
1024      * @return {@link Optional#isPresent()} is {@code true} if reading was
1025      *         successful and data exists in datastore; {@link Optional#isPresent()} is
1026      *         {@code false} otherwise
1027      */
1028     public static <T extends DataObject> Optional<T> readFromDs(LogicalDatastoreType store, InstanceIdentifier<T> path,
1029             ReadTransaction rTx) {
1030         CheckedFuture<Optional<T>, ReadFailedException> resultFuture = rTx.read(store, path);
1031         try {
1032             return resultFuture.checkedGet();
1033         } catch (ReadFailedException e) {
1034             LOG.warn("Read failed from DS.", e);
1035             return Optional.absent();
1036         }
1037     }
1038
1039     static byte[] bytesFromHexString(String values) {
1040         String target = "";
1041         if (values != null) {
1042             target = values;
1043         }
1044         String[] octets = target.split(":");
1045
1046         byte[] ret = new byte[octets.length];
1047         for (int i = 0; i < octets.length; i++) {
1048             ret[i] = Integer.valueOf(octets[i], 16).byteValue();
1049         }
1050         return ret;
1051     }
1052 }