renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java

   1 /*
   2  * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow;
  10
  11 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP;
  12 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4;
  13 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6;
  14 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch;
  15 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns;
  16 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.createNodePath;
  17 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction;
  18 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch;
  19 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum;
  20 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.gotoTableIns;
  21 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.groupAction;
  22 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.instructions;
  23 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpOpAction;
  24 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpShaAction;
  25 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction;
  26 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction;
  27 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action;
  28 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction;
  29 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction;
  30 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction;
  31 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction;
  32 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction;
  33 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction;
  34 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction;
  35
  36 import java.math.BigInteger;
  37 import java.util.ArrayList;
  38 import java.util.Collection;
  39 import java.util.Collections;
  40 import java.util.HashMap;
  41 import java.util.HashSet;
  42 import java.util.List;
  43 import java.util.Map;
  44 import java.util.Map.Entry;
  45 import java.util.Objects;
  46 import java.util.Set;
  47
  48 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  49 import org.opendaylight.controller.md.sal.binding.api.ReadTransaction;
  50 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  51 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  52 import org.opendaylight.groupbasedpolicy.endpoint.EpKey;
  53 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext;
  54 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.PolicyManager.FlowMap;
  55 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch;
  56 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals;
  57 import org.opendaylight.groupbasedpolicy.resolver.EgKey;
  58 import org.opendaylight.groupbasedpolicy.resolver.PolicyInfo;
  59 import org.opendaylight.groupbasedpolicy.resolver.TenantUtils;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix;
  63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.EndpointLocation.LocationType;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.L3Context;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.Subnet;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.ethernet.match.fields.EthernetDestinationBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatchBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.Layer3Match;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7;
 102 import org.opendaylight.yangtools.yang.binding.DataObject;
 103 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 104 import org.slf4j.Logger;
 105 import org.slf4j.LoggerFactory;
 106
 107 import com.google.common.base.Optional;
 108 import com.google.common.collect.HashMultimap;
 109 import com.google.common.collect.SetMultimap;
 110 import com.google.common.collect.Sets;
 111 import com.google.common.util.concurrent.CheckedFuture;
 112
 113 /**
 114  * Manage the table that maps the destination address to the next hop for the
 115  * path as well as applies any relevant routing transformations.
 116  */
 117 public class DestinationMapper extends FlowTable {
 118
 119     protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class);
 120
 121     // TODO Li alagalah: Improve UT coverage for this class.
 122
 123     // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows
 124     // This ensures we have the appropriate network-containment'
 125
 126     public static final short TABLE_ID = 2;
 127     /**
 128      * This is the MAC address of the magical router in the sky
 129      */
 130     public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5");
 131     public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00");
 132
 133     public DestinationMapper(OfContext ctx) {
 134         super(ctx);
 135     }
 136
 137     Map<TenantId, HashSet<Subnet>> subnetsByTenant = new HashMap<TenantId, HashSet<Subnet>>();
 138
 139     @Override
 140     public short getTableId() {
 141         return TABLE_ID;
 142     }
 143
 144     @Override
 145     public void sync(NodeId nodeId, PolicyInfo policyInfo, FlowMap flowMap) throws Exception {
 146
 147         TenantId currentTenant;
 148
 149         flowMap.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null));
 150
 151         SetMultimap<EpKey, EpKey> visitedEps = HashMultimap.create();
 152         Set<EndpointFwdCtxOrdinals> epOrdSet = new HashSet<>();
 153
 154         for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) {
 155             Set<EndpointGroupId> srcEpgIds = new HashSet<>();
 156             if (srcEp.getEndpointGroup() != null)
 157                 srcEpgIds.add(srcEp.getEndpointGroup());
 158             if (srcEp.getEndpointGroups() != null)
 159                 srcEpgIds.addAll(srcEp.getEndpointGroups());
 160
 161             for (EndpointGroupId epgId : srcEpgIds) {
 162                 EgKey epg = new EgKey(srcEp.getTenant(), epgId);
 163                 Set<EgKey> peers = Sets.union(Collections.singleton(epg), policyInfo.getPeers(epg));
 164                 for (EgKey peer : peers) {
 165                     for (Endpoint peerEp : ctx.getEndpointManager().getEndpointsForGroup(peer)) {
 166                         currentTenant = peerEp.getTenant();
 167                         subnetsByTenant.put(currentTenant, getSubnets(currentTenant));
 168                         EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress());
 169                         EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress());
 170
 171                         if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) {
 172                             continue;
 173                         }
 174                         syncEP(flowMap, nodeId, policyInfo, srcEp, peerEp);
 175                         visitedEps.put(srcEpKey, peerEpKey);
 176
 177                         // Process subnets and flood-domains for epPeer
 178                         EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo,
 179                                 peerEp);
 180                         epOrdSet.add(epOrds);
 181                     }
 182                 }
 183             }
 184         }
 185
 186         for (Entry<TenantId, HashSet<Subnet>> subnetEntry : subnetsByTenant.entrySet()) {
 187             if (subnetEntry.getValue() == null) {
 188                 LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey());
 189                 continue;
 190             }
 191             currentTenant = subnetEntry.getKey();
 192             for (Subnet sn : subnetEntry.getValue()) {
 193                 L3Context l3c = getL3ContextForSubnet(currentTenant, sn);
 194                 Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn,
 195                         OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId()));
 196                 if (arpFlow != null) {
 197                     flowMap.writeFlow(nodeId, TABLE_ID, arpFlow);
 198                 } else {
 199                     LOG.debug(
 200                             "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .",
 201                             sn.getIpPrefix().getValue());
 202                 }
 203             }
 204         }
 205
 206         // Write broadcast flows per flood domain.
 207         for (EndpointFwdCtxOrdinals epOrd : epOrdSet) {
 208             if (groupExists(nodeId, epOrd.getFdId())) {
 209                 flowMap.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd));
 210             }
 211         }
 212     }
 213
 214     // set up next-hop destinations for all the endpoints in the endpoint
 215     // group on the node
 216
 217     private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) {
 218         FlowId flowId = new FlowId(new StringBuilder().
 219                                     append("broadcast|").
 220                                     append(epOrd.getFdId()).
 221                                     toString());
 222         MatchBuilder mb = new MatchBuilder()
 223                             .setEthernetMatch(new EthernetMatchBuilder()
 224                             .setEthernetDestination(new EthernetDestinationBuilder().
 225                                                         setAddress(MULTICAST_MAC)
 226                                                         .setMask(MULTICAST_MAC).build())
 227                             .build());
 228         addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId())));
 229
 230         FlowBuilder flowb = base().setPriority(Integer.valueOf(140))
 231             .setId(flowId)
 232             .setMatch(mb.build())
 233             .setInstructions(
 234                     instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false),
 235                             groupAction(Long.valueOf(epOrd.getFdId())))));
 236         return flowb.build();
 237     }
 238
 239     private boolean groupExists(NodeId nodeId, Integer fdId) throws Exception {
 240         // Fetch existing GroupTables
 241         if (ctx.getDataBroker() == null) {
 242             return false;
 243         }
 244
 245         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 246         InstanceIdentifier<Node> niid = createNodePath(nodeId);
 247         Optional<Node> r = t.read(LogicalDatastoreType.CONFIGURATION, niid).get();
 248         if (!r.isPresent())
 249             return false;
 250         FlowCapableNode fcn = r.get().getAugmentation(FlowCapableNode.class);
 251         if (fcn == null)
 252             return false;
 253
 254         if (fcn.getGroup() != null) {
 255             for (Group g : fcn.getGroup()) {
 256                 if (g.getGroupId().getValue().equals(Long.valueOf(fdId))) { // Group
 257                                                                             // Exists.
 258                     return true;
 259                 }
 260             }
 261         }
 262         return false;
 263     }
 264
 265     private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) {
 266
 267         if (ctx.getDataBroker() == null) {
 268             return null;
 269         }
 270
 271         MacAddress defaultMacAddress = ROUTER_MAC;
 272
 273         EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId());
 274         InstanceIdentifier<EndpointL3> endpointsIid = InstanceIdentifier.builder(Endpoints.class)
 275             .child(EndpointL3.class, l3Key)
 276             .build();
 277         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 278
 279         Optional<EndpointL3> r;
 280         try {
 281             r = t.read(LogicalDatastoreType.OPERATIONAL, endpointsIid).get();
 282             if (!r.isPresent())
 283                 return defaultMacAddress;
 284             EndpointL3 epL3 = r.get();
 285             if (epL3.getMacAddress() == null) {
 286                 return defaultMacAddress;
 287             } else {
 288                 return epL3.getMacAddress();
 289             }
 290         } catch (Exception e) {
 291             LOG.error("Error reading EndpointL3 {}.{}", l3c, ipAddress, e);
 292             return null;
 293         }
 294     }
 295
 296     private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) {
 297         L3Context l3c = ctx.getPolicyResolver().getTenant(tenantId).resolveL3Context(sn.getId());
 298         return l3c;
 299     }
 300
 301     private Flow createRouterArpFlow(TenantId tenantId, NodeId nodeId, Subnet sn, int l3Id) {
 302         if (sn == null || sn.getVirtualRouterIp() == null) {
 303             LOG.trace("Didn't create routerArpFlow since either subnet or subnet virtual router was null");
 304             return null;
 305         }
 306         /*
 307          * TODO: Li alagalah: This should be new Yang "gateways" list as well,
 308          * that expresses the gateway and prefixes it is interface for. Should
 309          * also check for external.
 310          */
 311         if (sn.getVirtualRouterIp().getIpv4Address() != null) {
 312             String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue();
 313
 314             L3Context l3c = getL3ContextForSubnet(tenantId, sn);
 315             if (l3c == null) {
 316                 LOG.error("No L3 Context found associated with subnet {}", sn.getId());
 317             }
 318
 319             MacAddress routerMac = routerPortMac(l3c, sn.getVirtualRouterIp());
 320             if (routerMac == null) {
 321                 return null;
 322             }
 323
 324             BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue()));
 325
 326             FlowId flowId = new FlowId(new StringBuffer().append("routerarp|")
 327                 .append(sn.getId().getValue())
 328                 .append("|")
 329                 .append(ikey)
 330                 .append("|")
 331                 .append(l3Id)
 332                 .toString());
 333             MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match(
 334                     new ArpMatchBuilder().setArpOp(Integer.valueOf(1))
 335                         .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32"))
 336                         .build());
 337             addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id)));
 338
 339             FlowBuilder flowb = base().setPriority(150)
 340                 .setId(flowId)
 341                 .setMatch(mb.build())
 342                 .setInstructions(
 343                         instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac),
 344                                 nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(),
 345                                 nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(),
 346                                 nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue()
 347                                         + ":INPORT")))));
 348             return flowb.build();
 349         } else {
 350             LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId()
 351                 .getValue());
 352             return null;
 353         }
 354
 355     }
 356
 357     private Flow createLocalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) {
 358
 359         // TODO Li alagalah - refactor common code but keep simple method
 360         ArrayList<Instruction> instructions = new ArrayList<>();
 361         List<Action> applyActions = new ArrayList<>();
 362
 363         int order = 0;
 364
 365         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 366         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 367         Action setNextHop;
 368         String nextHop;
 369
 370         // BEGIN L2 LOCAL
 371         nextHop = ofc.getNodeConnectorId().getValue();
 372
 373         long portNum;
 374         try {
 375             portNum = getOfPortNum(ofc.getNodeConnectorId());
 376         } catch (NumberFormatException ex) {
 377             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 378             return null;
 379         }
 380
 381         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 382
 383         // END L2 LOCAL
 384
 385         order += 1;
 386         applyActions.add(setdEPG);
 387         applyActions.add(setdCG);
 388         applyActions.add(setNextHop);
 389         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 390             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 391             .build();
 392         instructions.add(applyActionsIns);
 393
 394         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 395             .setInstruction(gotoTableIns((short) (getTableId() + 1)))
 396             .build();
 397         instructions.add(gotoTable);
 398
 399         FlowId flowid = new FlowId(new StringBuilder().append(epFwdCtxOrds.getBdId())
 400             .append("|l2|")
 401             .append(ep.getMacAddress().getValue())
 402             .append("|")
 403             .append(nextHop)
 404             .toString());
 405         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 406         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
 407         FlowBuilder flowb = base().setId(flowid)
 408             .setPriority(Integer.valueOf(50))
 409             .setMatch(mb.build())
 410             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 411         return flowb.build();
 412     }
 413
 414     private void syncEP(FlowMap flowMap, NodeId nodeId, PolicyInfo policyInfo, Endpoint srcEp, Endpoint destEp)
 415             throws Exception {
 416
 417         // TODO: Conditions messed up, but for now, send policyInfo until this
 418         // is fixed.
 419         EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, destEp);
 420         EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, srcEp);
 421
 422         if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) {
 423             LOG.trace("Didn't process endpoint due to either tenant, or EPG(s) being null", destEp.getKey());
 424             return;
 425         }
 426         OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class);
 427
 428         // ////////////////////////////////////////////////////////////////////////////////////////
 429         /*
 430          * NOT HANDLING EXTERNALS TODO: alagalah Li: External Gateway
 431          * functionality needed here.
 432          */
 433         if (LocationType.External.equals(ofc.getLocationType())) {
 434             // XXX - TODO - perform NAT and send to the external network
 435             // TODO: Use case neutron gateway interface
 436             LOG.warn("External endpoints not yet supported");
 437             return;
 438         }
 439
 440         /*
 441          * Only care about subnets for L3, but fetch them before loop. We need
 442          * the local subnets for setting SRC MAC for routing. All Routing is now
 443          * done locally! YAY! Instead of being shovelled L2 style across network
 444          * ala Helium.
 445          */
 446         List<Subnet> localSubnets = getLocalSubnets(nodeId);
 447         if (localSubnets == null) {
 448             LOG.error("No subnets could be found locally for node: {}", nodeId);
 449             return;
 450         }
 451
 452         if (Objects.equals(ofc.getNodeId(), nodeId)) {
 453             // this is a local endpoint; send to the approppriate local
 454             // port
 455
 456             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 457                 flowMap.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc));
 458             }
 459             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 460             // The Endpoint conflation must end!
 461             if (destEp.getL3Address() == null) {
 462                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 463                 return;
 464             }
 465
 466             for (L3Address l3a : destEp.getL3Address()) {
 467                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 468                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 469                             destEp.getL3Address());
 470                     continue;
 471                 } else {
 472                     for (Subnet localSubnet : localSubnets) {
 473                         Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet);
 474                         if (flow != null) {
 475                             flowMap.writeFlow(nodeId, TABLE_ID, flow);
 476                         } else {
 477                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 478                                     localSubnet.getIpPrefix().getValue());
 479                         }
 480                     }
 481                 }
 482             }
 483         } else {
 484             // this endpoint is on a different switch; send to the
 485             // appropriate tunnel
 486             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 487                 Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc);
 488                 if (remoteL2Flow != null) {
 489                     flowMap.writeFlow(nodeId, TABLE_ID, remoteL2Flow);
 490                 }
 491             } else {
 492                 LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}",
 493                         srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId());
 494             }
 495
 496             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 497             // The Endpoint conflation must end!
 498             if (destEp.getL3Address() == null) {
 499                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 500                 return;
 501             }
 502             for (L3Address l3a : destEp.getL3Address()) {
 503                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 504                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 505                             destEp.getL3Address());
 506                     continue;
 507                 } else {
 508                     for (Subnet localSubnet : localSubnets) {
 509                         Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds,
 510                                 destEpFwdCtxOrds, ofc, localSubnet);
 511                         if (remoteL3Flow != null) {
 512                             flowMap.writeFlow(nodeId, TABLE_ID, remoteL3Flow);
 513                         } else {
 514                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 515                                     localSubnet.getIpPrefix().getValue());
 516                         }
 517                     }
 518                 }
 519             }
 520         } // remote (tunnel)
 521
 522         // }
 523
 524     }
 525
 526     /*
 527      * ################################## DestMapper Flow methods
 528      * ##################################
 529      */
 530     private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
 531             OfOverlayContext ofc, Subnet srcSubnet) {
 532
 533         // TODO Li alagalah - refactor common code but keep simple method
 534
 535         Subnet destSubnet = null;
 536         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 537         if (subnets == null) {
 538             LOG.trace("No subnets in tenant {}", destL3Address.getIpAddress());
 539             return null;
 540         }
 541         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 542         for (Subnet subnet : subnets) {
 543             // TODO Li alagalah add IPv6 support
 544             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 545                 destSubnet = subnet;
 546                 break;
 547             }
 548         }
 549         if (destSubnet == null) {
 550             LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 551             return null;
 552         }
 553
 554         if (destSubnet.getVirtualRouterIp() == null) {
 555             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 556                     destL3Address.getKey());
 557             return null;
 558         }
 559
 560         if (srcSubnet.getVirtualRouterIp() == null) {
 561             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 562             return null;
 563         }
 564         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 565         if (destL3c == null || destL3c.getId() == null) {
 566             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 567             return null;
 568         }
 569         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 570         if (srcL3c == null || srcL3c.getId() == null) {
 571             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 572             return null;
 573         }
 574
 575         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 576             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 577                 .getValue(), destL3c.getId().getValue());
 578             return null;
 579         }
 580
 581         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 582         MacAddress epDestMac = destEp.getMacAddress();
 583         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 584
 585         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 586             // This is our final destination, so match on actual EP mac.
 587             matcherMac = epDestMac;
 588         }
 589
 590         ArrayList<Instruction> l3instructions = new ArrayList<>();
 591         List<Action> applyActions = new ArrayList<>();
 592         List<Action> l3ApplyActions = new ArrayList<>();
 593
 594         int order = 0;
 595
 596         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 597         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 598         Action setNextHop;
 599         String nextHop;
 600
 601         // BEGIN L3 LOCAL
 602         nextHop = ofc.getNodeConnectorId().getValue();
 603
 604         long portNum;
 605         try {
 606             portNum = getOfPortNum(ofc.getNodeConnectorId());
 607         } catch (NumberFormatException ex) {
 608             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 609             return null;
 610         }
 611
 612         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 613         // END L3 LOCAL
 614
 615         // Lets not re-write the srcMac if its local.
 616         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
 617             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
 618             l3ApplyActions.add(setDlSrc);
 619         }
 620
 621         Action setDlDst = setDlDstAction(epDestMac);
 622         l3ApplyActions.add(setDlDst);
 623
 624         Action decTtl = decNwTtlAction();
 625         l3ApplyActions.add(decTtl);
 626
 627         order += 1;
 628         applyActions.add(setdEPG);
 629         applyActions.add(setdCG);
 630         applyActions.add(setNextHop);
 631
 632         applyActions.addAll(l3ApplyActions);
 633         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 634             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 635             .build();
 636
 637         l3instructions.add(applyActionsIns);
 638         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 639             .setInstruction(gotoTableIns((short) (getTableId() + 1)))
 640             .build();
 641         l3instructions.add(gotoTable);
 642         Layer3Match m = null;
 643         Long etherType = null;
 644         String ikey = null;
 645         if (destL3Address.getIpAddress().getIpv4Address() != null) {
 646             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
 647             etherType = IPv4;
 648             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 649         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
 650             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
 651             etherType = IPv6;
 652             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 653         } else {
 654             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
 655             return null;
 656         }
 657
 658         FlowId flowid = new FlowId(new StringBuilder().append(Integer.toString(epFwdCtxOrds.getL3Id()))
 659             .append("|l3|")
 660             .append(ikey)
 661             .append("|")
 662             .append(Integer.toString(epFwdCtxOrds.getEpgId()))
 663             .append("|")
 664             .append(Integer.toString(epFwdCtxOrds.getCgId()))
 665             .append("|")
 666             .append(matcherMac)
 667             .append("|")
 668             .append(destSubnetGatewayMac)
 669             .append("|")
 670             .append(nextHop)
 671             .toString());
 672         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
 673             .setLayer3Match(m);
 674         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 675         FlowBuilder flowb = base().setId(flowid)
 676             .setPriority(Integer.valueOf(132))
 677             .setMatch(mb.build())
 678             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 679         return flowb.build();
 680     }
 681
 682     private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds,
 683             EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) {
 684
 685         // TODO Li alagalah - refactor common code but keep simple method
 686
 687         // this endpoint is on a different switch; send to the
 688         // appropriate tunnel
 689
 690         ArrayList<Instruction> instructions = new ArrayList<>();
 691         List<Action> applyActions = new ArrayList<>();
 692
 693         int order = 0;
 694
 695         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
 696         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
 697         Action setNextHop;
 698         String nextHop;
 699
 700         // BEGIN TUNNEL HANDLING
 701         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId());
 702         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId);
 703         if (tunDst == null) {
 704             LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep);
 705             return null;
 706         }
 707         if (tunPort == null) {
 708             LOG.warn("Failed to get Tunnel Port for NodeId {} with EP {}", nodeId, ep);
 709             return null;
 710         }
 711
 712         Action tundstAction;
 713
 714         if (tunDst.getIpv4Address() != null) {
 715             nextHop = tunDst.getIpv4Address().getValue();
 716             tundstAction = nxLoadTunIPv4Action(nextHop, false);
 717         } else if (tunDst.getIpv6Address() != null) {
 718             // nextHop = tunDst.getIpv6Address().getValue();
 719             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
 720                     ofc.getNodeId());
 721             return null;
 722         } else {
 723             // this shouldn't happen
 724             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
 725             return null;
 726         }
 727
 728         long portNum;
 729         try {
 730             portNum = getOfPortNum(tunPort);
 731         } catch (NumberFormatException ex) {
 732             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 733             return null;
 734         }
 735
 736         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 737         applyActions.add(tundstAction);
 738         // END TUNNEL
 739
 740         order += 1;
 741         applyActions.add(setdEPG);
 742         applyActions.add(setdCG);
 743         applyActions.add(setNextHop);
 744         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 745             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 746             .build();
 747         instructions.add(applyActionsIns);
 748
 749         applyActionsIns = new InstructionBuilder().setOrder(order++)
 750             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 751             .build();
 752
 753         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 754             .setInstruction(gotoTableIns((short) (getTableId() + 1)))
 755             .build();
 756         instructions.add(gotoTable);
 757
 758         FlowId flowid = new FlowId(new StringBuilder().append(destEpFwdCtxOrds.getBdId())
 759             .append("|l2|")
 760             .append(ep.getMacAddress().getValue())
 761             .append("|")
 762             .append(srcEpFwdCtxOrds.getBdId())
 763             .append("|")
 764             .append(nextHop)
 765             .toString());
 766         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 767         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId())));
 768         FlowBuilder flowb = base().setId(flowid)
 769             .setPriority(Integer.valueOf(50))
 770             .setMatch(mb.build())
 771             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 772
 773         return flowb.build();
 774     }
 775
 776     private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId,
 777             EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc,
 778             Subnet srcSubnet) {
 779
 780         // TODO Li alagalah - refactor common code but keep simple method
 781
 782         // this endpoint is on a different switch; send to the
 783         // appropriate tunnel
 784         Subnet destSubnet = null;
 785         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 786         if (subnets == null) {
 787             LOG.trace("No subnets in tenant {}", destL3Address.getIpAddress());
 788             return null;
 789         }
 790         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 791         for (Subnet subnet : subnets) {
 792             // TODO Li alagalah add IPv6 support
 793             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 794                 destSubnet = subnet;
 795                 break;
 796             }
 797         }
 798         if (destSubnet == null) {
 799             LOG.info("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 800             return null;
 801         }
 802
 803         if (destSubnet.getVirtualRouterIp() == null) {
 804             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 805                     destL3Address.getKey());
 806             return null;
 807         }
 808
 809         if (srcSubnet.getVirtualRouterIp() == null) {
 810             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 811             return null;
 812         }
 813         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 814         if (destL3c == null || destL3c.getId() == null) {
 815             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 816             return null;
 817         }
 818         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 819         if (srcL3c == null || srcL3c.getId() == null) {
 820             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 821             return null;
 822         }
 823
 824         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 825             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 826                 .getValue(), destL3c.getId().getValue());
 827             return null;
 828         }
 829
 830         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 831         MacAddress epDestMac = destEp.getMacAddress();
 832         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 833
 834         ArrayList<Instruction> l3instructions = new ArrayList<>();
 835         List<Action> applyActions = new ArrayList<>();
 836         List<Action> l3ApplyActions = new ArrayList<>();
 837
 838         int order = 0;
 839
 840         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
 841         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
 842         Action setNextHop;
 843         String nextHop;
 844
 845         // BEGIN TUNNEL HANDLING
 846         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId());
 847         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId);
 848         if (tunDst == null) {
 849             LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address);
 850             return null;
 851         }
 852         if (tunPort == null) {
 853             LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, destL3Address);
 854             return null;
 855         }
 856
 857         Action tundstAction;
 858
 859         if (tunDst.getIpv4Address() != null) {
 860             nextHop = tunDst.getIpv4Address().getValue();
 861             tundstAction = nxLoadTunIPv4Action(nextHop, false);
 862         } else if (tunDst.getIpv6Address() != null) {
 863             // nextHop = tunDst.getIpv6Address().getValue();
 864             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
 865                     ofc.getNodeId());
 866             return null;
 867         } else {
 868             // this shouldn't happen
 869             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
 870             return null;
 871         }
 872
 873         long portNum;
 874         try {
 875             portNum = getOfPortNum(tunPort);
 876         } catch (NumberFormatException ex) {
 877             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 878             return null;
 879         }
 880
 881         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 882         applyActions.add(tundstAction);
 883         // END TUNNEL
 884
 885         order += 1;
 886         applyActions.add(setdEPG);
 887         applyActions.add(setdCG);
 888         applyActions.add(setNextHop);
 889
 890         Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
 891         l3ApplyActions.add(setDlSrc);
 892
 893         Action setDlDst = setDlDstAction(epDestMac);
 894         l3ApplyActions.add(setDlDst);
 895
 896         Action decTtl = decNwTtlAction();
 897         l3ApplyActions.add(decTtl);
 898
 899         applyActions.addAll(l3ApplyActions);
 900         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 901             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 902             .build();
 903
 904         l3instructions.add(applyActionsIns);
 905         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 906             .setInstruction(gotoTableIns((short) (getTableId() + 1)))
 907             .build();
 908         l3instructions.add(gotoTable);
 909         Layer3Match m = null;
 910         Long etherType = null;
 911         String ikey = null;
 912         if (destL3Address.getIpAddress().getIpv4Address() != null) {
 913             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
 914             etherType = IPv4;
 915             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 916         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
 917             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
 918             etherType = IPv6;
 919             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 920         } else {
 921             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
 922             return null;
 923         }
 924
 925         FlowId flowid = new FlowId(new StringBuilder().append(Integer.toString(destEpFwdCtxOrds.getL3Id()))
 926             .append("|l3|")
 927             .append(ikey)
 928             .append("|")
 929             .append(matcherMac)
 930             .append("|")
 931             .append(destSubnetGatewayMac)
 932             .append("|")
 933             .append(srcEpFwdCtxOrds.getL3Id())
 934             .append("|")
 935             .append(nextHop)
 936             .toString());
 937         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
 938             .setLayer3Match(m);
 939         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id())));
 940
 941         FlowBuilder flowb = base().setId(flowid)
 942             .setPriority(Integer.valueOf(132))
 943             .setMatch(mb.build())
 944             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 945         return flowb.build();
 946     }
 947
 948     private NetworkDomainId getEPNetworkContainment(Endpoint endpoint) {
 949         if (endpoint.getNetworkContainment() != null) {
 950             return endpoint.getNetworkContainment();
 951         } else {
 952             /*
 953              * TODO: Be alagalah: Endpoint Refactor: This should be set on input
 954              * which we can't do because of the backwards way endpoints were
 955              * "architected".
 956              */
 957             return ctx.getPolicyResolver()
 958                 .getTenant(endpoint.getTenant())
 959                 .getEndpointGroup(endpoint.getEndpointGroup())
 960                 .getNetworkDomain();
 961         }
 962     }
 963
 964     private HashSet<Subnet> getSubnets(final TenantId tenantId) {
 965
 966         // if (subnetsByTenant.get(tenantId) != null) {
 967         // return subnetsByTenant.get(tenantId);
 968         // }
 969
 970         if (ctx.getDataBroker() == null) {
 971             return null;
 972         }
 973
 974         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 975         InstanceIdentifier<Tenant> tiid = TenantUtils.tenantIid(tenantId);
 976         Optional<Tenant> tenantInfo;
 977         try {
 978             tenantInfo = t.read(LogicalDatastoreType.CONFIGURATION, tiid).get();
 979         } catch (Exception e) {
 980             LOG.error("Could not read Tenant {}", tenantId, e);
 981             return null;
 982         }
 983
 984         HashSet<Subnet> subnets = new HashSet<Subnet>();
 985
 986         if (!tenantInfo.isPresent()) {
 987             LOG.warn("Tenant {} not found", tenantId);
 988             return null;
 989         }
 990
 991         subnets.addAll(tenantInfo.get().getSubnet());
 992         // subnetsByTenant.put(tenantId, subnets);
 993         return subnets;
 994     }
 995
 996     // Need a method to get subnets for EPs attached to the node locally
 997     // to set the source Mac address for the router interface.
 998     private List<Subnet> getLocalSubnets(NodeId nodeId) {
 999         Collection<Endpoint> endpointsForNode = ctx.getEndpointManager().getEndpointsForNode(nodeId);
1000
1001         List<Subnet> localSubnets = new ArrayList<Subnet>();
1002
1003         for (Endpoint endpoint : endpointsForNode) {
1004             HashSet<Subnet> subnets = getSubnets(endpoint.getTenant());
1005             if (subnets == null) {
1006                 LOG.error("No local subnets.");
1007                 return null;
1008             }
1009             NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint);
1010             for (Subnet subnet : subnets) {
1011                 if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) {
1012                     localSubnets.add(subnet);
1013                 }
1014             }
1015         }
1016         return localSubnets;
1017     }
1018
1019     /**
1020      * Reads data from datastore as synchronous call.
1021      *
1022      * @return {@link Optional#isPresent()} is {@code true} if reading was
1023      *         successful and data exists in datastore; {@link Optional#isPresent()} is
1024      *         {@code false} otherwise
1025      */
1026     public static <T extends DataObject> Optional<T> readFromDs(LogicalDatastoreType store, InstanceIdentifier<T> path,
1027             ReadTransaction rTx) {
1028         CheckedFuture<Optional<T>, ReadFailedException> resultFuture = rTx.read(store, path);
1029         try {
1030             return resultFuture.checkedGet();
1031         } catch (ReadFailedException e) {
1032             LOG.warn("Read failed from DS.", e);
1033             return Optional.absent();
1034         }
1035     }
1036
1037     static byte[] bytesFromHexString(String values) {
1038         String target = "";
1039         if (values != null) {
1040             target = values;
1041         }
1042         String[] octets = target.split(":");
1043
1044         byte[] ret = new byte[octets.length];
1045         for (int i = 0; i < octets.length; i++) {
1046             ret[i] = Integer.valueOf(octets[i], 16).byteValue();
1047         }
1048         return ret;
1049     }
1050 }