renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java

   1 /*
   2  * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow;
  10
  11 import com.google.common.base.Optional;
  12 import com.google.common.base.Strings;
  13 import com.google.common.collect.HashMultimap;
  14 import com.google.common.collect.SetMultimap;
  15 import com.google.common.collect.Sets;
  16 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  17 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  18 import org.opendaylight.groupbasedpolicy.endpoint.EpKey;
  19 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext;
  20 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.PolicyManager.FlowMap;
  21 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch;
  22 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals;
  23 import org.opendaylight.groupbasedpolicy.resolver.EgKey;
  24 import org.opendaylight.groupbasedpolicy.resolver.IndexedTenant;
  25 import org.opendaylight.groupbasedpolicy.resolver.PolicyInfo;
  26 import org.opendaylight.groupbasedpolicy.resolver.TenantUtils;
  27 import org.opendaylight.groupbasedpolicy.util.IidFactory;
  28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  29 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
  30 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix;
  31 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.l3.prefix.fields.EndpointL3Gateways;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Prefix;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.EndpointLocation.LocationType;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.L3Context;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.Subnet;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.ethernet.match.fields.EthernetDestinationBuilder;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatchBuilder;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.Layer3Match;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
  74 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  75 import org.slf4j.Logger;
  76 import org.slf4j.LoggerFactory;
  77
  78 import java.math.BigInteger;
  79 import java.util.ArrayList;
  80 import java.util.Collection;
  81 import java.util.Collections;
  82 import java.util.HashMap;
  83 import java.util.HashSet;
  84 import java.util.List;
  85 import java.util.Map;
  86 import java.util.Map.Entry;
  87 import java.util.Objects;
  88 import java.util.Set;
  89
  90 import static com.google.common.base.Preconditions.checkNotNull;
  91 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP;
  92 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4;
  93 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6;
  94 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch;
  95 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns;
  96 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.createNodePath;
  97 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction;
  98 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch;
  99 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum;
 100 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.gotoTableIns;
 101 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.groupAction;
 102 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.instructions;
 103 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpOpAction;
 104 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpShaAction;
 105 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction;
 106 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction;
 107 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action;
 108 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction;
 109 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction;
 110 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction;
 111 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction;
 112 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction;
 113 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction;
 114 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction;
 115 import static org.opendaylight.groupbasedpolicy.util.DataStoreHelper.readFromDs;
 116
 117 /**
 118  * Manage the table that maps the destination address to the next hop for the
 119  * path as well as applies any relevant routing transformations.
 120  */
 121 public class DestinationMapper extends FlowTable {
 122
 123     protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class);
 124
 125     // TODO Li alagalah: Improve UT coverage for this class.
 126
 127     // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows
 128     // This ensures we have the appropriate network-containment'
 129
 130     public static short TABLE_ID;
 131     /**
 132      * This is the MAC address of the magical router in the sky
 133      */
 134     public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5");
 135     public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00");
 136     public static final Integer BASE_L3_PRIORITY = 100;
 137
 138     public DestinationMapper(OfContext ctx, short tableId) {
 139         super(ctx);
 140         this.TABLE_ID = tableId;
 141     }
 142
 143     Map<TenantId, HashSet<Subnet>> subnetsByTenant = new HashMap<TenantId, HashSet<Subnet>>();
 144
 145     @Override
 146     public short getTableId() {
 147         return TABLE_ID;
 148     }
 149
 150     @Override
 151     public void sync(NodeId nodeId, PolicyInfo policyInfo, FlowMap flowMap) throws Exception {
 152
 153         TenantId currentTenant = null;
 154
 155         flowMap.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null, TABLE_ID));
 156
 157         SetMultimap<EpKey, EpKey> visitedEps = HashMultimap.create();
 158         Set<EndpointFwdCtxOrdinals> epOrdSet = new HashSet<>();
 159
 160         for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) {
 161             Set<EndpointGroupId> srcEpgIds = new HashSet<>();
 162             if (srcEp.getEndpointGroup() != null)
 163                 srcEpgIds.add(srcEp.getEndpointGroup());
 164             if (srcEp.getEndpointGroups() != null)
 165                 srcEpgIds.addAll(srcEp.getEndpointGroups());
 166
 167             for (EndpointGroupId epgId : srcEpgIds) {
 168                 EgKey epg = new EgKey(srcEp.getTenant(), epgId);
 169                 Set<EgKey> peers = Sets.union(Collections.singleton(epg), policyInfo.getPeers(epg));
 170                 for (EgKey peer : peers) {
 171                     for (Endpoint peerEp : ctx.getEndpointManager().getEndpointsForGroup(peer)) {
 172                         currentTenant = peerEp.getTenant();
 173                         subnetsByTenant.put(currentTenant, getSubnets(currentTenant));
 174                         EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress());
 175                         EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress());
 176
 177                         if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) {
 178                             continue;
 179                         }
 180                         syncEP(flowMap, nodeId, policyInfo, srcEp, peerEp);
 181                         visitedEps.put(srcEpKey, peerEpKey);
 182
 183                         // Process subnets and flood-domains for epPeer
 184                         EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo,
 185                                 peerEp);
 186                         if (epOrds == null) {
 187                             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", peerEp);
 188                             continue;
 189                         }
 190
 191                         epOrdSet.add(epOrds);
 192                     }
 193                 }
 194             }
 195         }
 196
 197         for (Entry<TenantId, HashSet<Subnet>> subnetEntry : subnetsByTenant.entrySet()) {
 198             if (subnetEntry.getValue() == null) {
 199                 LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey());
 200                 continue;
 201             }
 202             currentTenant = subnetEntry.getKey();
 203             for (Subnet sn : subnetEntry.getValue()) {
 204                 L3Context l3c = getL3ContextForSubnet(currentTenant, sn);
 205                 Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn,
 206                         OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId()));
 207                 if (arpFlow != null) {
 208                     flowMap.writeFlow(nodeId, TABLE_ID, arpFlow);
 209                 } else {
 210                     LOG.debug(
 211                             "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .",
 212                             sn.getIpPrefix().getValue());
 213                 }
 214             }
 215         }
 216
 217         // Write broadcast flows per flood domain.
 218         for (EndpointFwdCtxOrdinals epOrd : epOrdSet) {
 219             if (groupExists(nodeId, epOrd.getFdId())) {
 220                 flowMap.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd));
 221             }
 222         }
 223
 224         // L3 Prefix Endpoint handling
 225         Collection<EndpointL3Prefix> prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(currentTenant);
 226         if (prefixEps != null) {
 227             LOG.trace("DestinationMapper - Processing L3PrefixEndpoints");
 228             for (EndpointL3Prefix prefixEp : prefixEps) {
 229                 List<Subnet> localSubnets = getLocalSubnets(nodeId);
 230                 if (localSubnets == null) {
 231                     continue;
 232                 }
 233                 for (Subnet localSubnet: localSubnets) {
 234                     Flow prefixFlow = createL3PrefixFlow(prefixEp, policyInfo, nodeId, localSubnet);
 235                     if (prefixFlow != null) {
 236                         flowMap.writeFlow(nodeId, TABLE_ID, prefixFlow);
 237                         LOG.trace("Wrote L3Prefix flow");
 238                     }
 239                 }
 240             }
 241         }
 242     }
 243
 244     // set up next-hop destinations for all the endpoints in the endpoint
 245     // group on the node
 246
 247     private Flow createL3PrefixFlow(EndpointL3Prefix prefixEp, PolicyInfo policyInfo, NodeId nodeId, Subnet subnet) throws Exception {
 248         /*
 249          * Priority: 100+lengthprefix
 250          * Match: prefix, l3c, "mac address of router" ?
 251          * Action:
 252          * - set Reg2, Reg3 for L3Ep by L2Ep ?
 253          * - if external,
 254          * - Reg7: use switch location external port else punt for now
 255          * - if internal
 256          * - Reg7: grab L2Ep from L3Ep and use its location info
 257          * - goto_table: POLENF (will check there for external on EP)
 258          */
 259
 260         ReadOnlyTransaction rTx = ctx.getDataBroker().newReadOnlyTransaction();
 261         // TODO Bug #3440 Target: Be - should support for more than first gateway.
 262         EndpointL3Gateways l3Gateway = prefixEp.getEndpointL3Gateways().get(0);
 263         Optional<EndpointL3> optL3Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 264                 IidFactory.l3EndpointIid(l3Gateway.getL3Context(), l3Gateway.getIpAddress()), rTx);
 265         if (!optL3Ep.isPresent()) {
 266             LOG.error("createL3PrefixFlow - L3Endpoint gateway {} for L3Prefix {} not found.", l3Gateway, prefixEp);
 267             return null;
 268         }
 269         EndpointL3 l3Ep = optL3Ep.get();
 270         if (l3Ep.getL2Context() == null || l3Ep.getMacAddress() == null) {
 271             LOG.debug("L3 endpoint representing L3 gateway does not contain L2-context or MAC address. {}", l3Ep);
 272             return null;
 273         }
 274         Optional<Endpoint> optL2Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 275                 IidFactory.endpointIid(l3Ep.getL2Context(), l3Ep.getMacAddress()), rTx);
 276         if (!optL2Ep.isPresent()) {
 277             LOG.error("createL3PrefixFlow - L2Endpoint for L3Gateway {} not found.", l3Ep);
 278             return null;
 279         }
 280         Endpoint l2Ep = optL2Ep.get();
 281         EndpointFwdCtxOrdinals epFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, l2Ep);
 282         if (epFwdCtxOrds == null) {
 283             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", l2Ep);
 284             return null;
 285         }
 286
 287         NetworkDomainId epNetworkContainment = getEPNetworkContainment(l2Ep);
 288
 289         MacAddress epDestMac = l2Ep.getMacAddress();
 290         MacAddress destSubnetGatewayMac = l2Ep.getMacAddress();
 291         L3Context destL3c = getL3ContextForSubnet(prefixEp.getTenant(), subnet);
 292         if (destL3c == null || destL3c.getId() == null) {
 293             LOG.error("No L3 Context found associated with subnet {}", subnet.getId());
 294             return null;
 295         }
 296
 297         MacAddress matcherMac = routerPortMac(destL3c, subnet.getVirtualRouterIp());
 298
 299         ArrayList<Instruction> l3instructions = new ArrayList<>();
 300         List<Action> applyActions = new ArrayList<>();
 301         List<Action> l3ApplyActions = new ArrayList<>();
 302
 303         int order = 0;
 304
 305         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 306         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 307         Action setNextHop;
 308         String nextHop=null;
 309
 310         OfOverlayContext ofc = l2Ep.getAugmentation(OfOverlayContext.class);
 311         LocationType location;
 312
 313         if (ofc != null && ofc.getLocationType() != null) {
 314             location = ofc.getLocationType();
 315         } else if (ofc != null) {
 316             // Augmentation, but using default location
 317             location = LocationType.Internal;
 318         } else {
 319             LOG.info("createL3PrefixFlow - Endpoint {} had no augmentation.", l2Ep);
 320             return null;
 321         }
 322
 323         long portNum = -1;
 324
 325         if (location.equals(LocationType.Internal)) {
 326             checkNotNull(ofc.getNodeConnectorId());
 327             nextHop = ofc.getNodeConnectorId().getValue();
 328             try {
 329                 portNum = getOfPortNum(ofc.getNodeConnectorId());
 330             } catch (NumberFormatException ex) {
 331                 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 332                 return null;
 333             }
 334
 335         } else {
 336             // External
 337             Set<NodeConnectorId> externalPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
 338             checkNotNull(externalPorts);
 339             for (NodeConnectorId externalPort : externalPorts) {
 340                 // TODO Bug #3440 Target: Be - should support for more than first external port.
 341                 //TODO Bug 3546 - Difficult: External port is unrelated to Tenant, L3C, L2BD..
 342                 nextHop = externalPort.getValue();
 343                 try {
 344                     portNum = getOfPortNum(externalPort);
 345                 } catch (NumberFormatException ex) {
 346                     LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 347                     return null;
 348                 }
 349                 continue;
 350             }
 351         }
 352
 353         if (Strings.isNullOrEmpty(nextHop)
 354                 || portNum == -1) {
 355             LOG.error("createL3Prefix - Cannot find nodeConnectorId for {} for Prefix: ", l2Ep, prefixEp);
 356             return null;
 357         }
 358         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 359
 360         Action setDlDst = setDlDstAction(epDestMac);
 361         l3ApplyActions.add(setDlDst);
 362
 363         Action decTtl = decNwTtlAction();
 364         l3ApplyActions.add(decTtl);
 365
 366         order += 1;
 367         applyActions.add(setdEPG);
 368         applyActions.add(setdCG);
 369         applyActions.add(setNextHop);
 370
 371         applyActions.addAll(l3ApplyActions);
 372         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 373             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 374             .build();
 375
 376         l3instructions.add(applyActionsIns);
 377         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 378             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 379             .build();
 380         l3instructions.add(gotoTable);
 381
 382         Layer3Match m = null;
 383         Long etherType = null;
 384         String ikey = null;
 385         Integer prefixLength=0;
 386         if (prefixEp.getIpPrefix().getIpv4Prefix() != null) {
 387             ikey = prefixEp.getIpPrefix().getIpv4Prefix().getValue();
 388             etherType = IPv4;
 389             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv4Prefix().getValue().split("/")[1]);
 390             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 391         } else if (prefixEp.getIpPrefix().getIpv6Prefix() != null) {
 392             ikey = prefixEp.getIpPrefix().getIpv6Prefix().getValue();
 393             etherType = IPv6;
 394             /*
 395              *  This will result in flows with priority between 100-228, but since its matching on IPv6 prefix as well
 396              *  this shouldn't pose and issue, as the priority is more important within the address space of the matcher,
 397              *  even though technically flows are processed in priority order.
 398              */
 399
 400             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv6Prefix().getValue().split("/")[1]);
 401             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 402         } else {
 403             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", prefixEp);
 404             return null;
 405         }
 406
 407         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType));
 408         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 409         Match match = mb.build();
 410         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "L3prefix", match);
 411         FlowBuilder flowb = base().setId(flowid)
 412             .setPriority(Integer.valueOf(BASE_L3_PRIORITY+prefixLength))
 413             .setMatch(match)
 414             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 415         return flowb.build();
 416     }
 417
 418     private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) {
 419         MatchBuilder mb = new MatchBuilder()
 420                             .setEthernetMatch(new EthernetMatchBuilder()
 421                             .setEthernetDestination(new EthernetDestinationBuilder().
 422                                                         setAddress(MULTICAST_MAC)
 423                                                         .setMask(MULTICAST_MAC).build())
 424                             .build());
 425         addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId())));
 426
 427         Match match = mb.build();
 428         FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "broadcast", match);
 429         FlowBuilder flowb = base().setPriority(Integer.valueOf(140))
 430             .setId(flowId)
 431             .setMatch(match)
 432             .setInstructions(
 433                     instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false),
 434                             groupAction(Long.valueOf(epOrd.getFdId())))));
 435
 436         return flowb.build();
 437     }
 438
 439     private boolean groupExists(NodeId nodeId, Integer fdId) throws Exception {
 440         // Fetch existing GroupTables
 441         if (ctx.getDataBroker() == null) {
 442             return false;
 443         }
 444
 445         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 446         InstanceIdentifier<Node> niid = createNodePath(nodeId);
 447         Optional<Node> r = t.read(LogicalDatastoreType.CONFIGURATION, niid).get();
 448         if (!r.isPresent())
 449             return false;
 450         FlowCapableNode fcn = r.get().getAugmentation(FlowCapableNode.class);
 451         if (fcn == null)
 452             return false;
 453
 454         if (fcn.getGroup() != null) {
 455             for (Group g : fcn.getGroup()) {
 456                 if (g.getGroupId().getValue().equals(Long.valueOf(fdId))) { // Group
 457                                                                             // Exists.
 458                     return true;
 459                 }
 460             }
 461         }
 462         return false;
 463     }
 464
 465     private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) {
 466
 467         if (ctx.getDataBroker() == null) {
 468             return null;
 469         }
 470
 471         MacAddress defaultMacAddress = ROUTER_MAC;
 472
 473         EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId());
 474         InstanceIdentifier<EndpointL3> endpointsIid = InstanceIdentifier.builder(Endpoints.class)
 475             .child(EndpointL3.class, l3Key)
 476             .build();
 477         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 478
 479         Optional<EndpointL3> r;
 480         try {
 481             r = t.read(LogicalDatastoreType.OPERATIONAL, endpointsIid).get();
 482             if (!r.isPresent())
 483                 return defaultMacAddress;
 484             EndpointL3 epL3 = r.get();
 485             if (epL3.getMacAddress() == null) {
 486                 return defaultMacAddress;
 487             } else {
 488                 return epL3.getMacAddress();
 489             }
 490         } catch (Exception e) {
 491             LOG.error("Error reading EndpointL3 {}.{}", l3c, ipAddress, e);
 492             return null;
 493         }
 494     }
 495
 496     private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) {
 497         IndexedTenant indexedTenant = ctx.getPolicyResolver().getTenant(tenantId);
 498         if (indexedTenant == null) {
 499             LOG.debug("Tenant {} is null, cannot get L3 context", tenantId);
 500             return null;
 501         }
 502         L3Context l3c = indexedTenant.resolveL3Context(sn.getId());
 503         return l3c;
 504     }
 505
 506     private Flow createRouterArpFlow(TenantId tenantId, NodeId nodeId, Subnet sn, int l3Id) {
 507         if (sn == null || sn.getVirtualRouterIp() == null) {
 508             LOG.trace("Didn't create routerArpFlow since either subnet or subnet virtual router was null");
 509             return null;
 510         }
 511         /*
 512          * TODO: Li alagalah: This should be new Yang "gateways" list as well,
 513          * that expresses the gateway and prefixes it is interface for. Should
 514          * also check for external.
 515          */
 516         if (sn.getVirtualRouterIp().getIpv4Address() != null) {
 517             String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue();
 518
 519             L3Context l3c = getL3ContextForSubnet(tenantId, sn);
 520             if (l3c == null) {
 521                 LOG.error("No L3 Context found associated with subnet {}", sn.getId());
 522             }
 523
 524             MacAddress routerMac = routerPortMac(l3c, sn.getVirtualRouterIp());
 525             if (routerMac == null) {
 526                 return null;
 527             }
 528
 529             BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue()));
 530
 531             MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match(
 532                     new ArpMatchBuilder().setArpOp(Integer.valueOf(1))
 533                         .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32"))
 534                         .build());
 535             addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id)));
 536
 537             Match match = mb.build();
 538             FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "routerarp", match);
 539             FlowBuilder flowb = base().setPriority(150)
 540                 .setId(flowId)
 541                 .setMatch(match)
 542                 .setInstructions(
 543                         instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac),
 544                                 nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(),
 545                                 nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(),
 546                                 nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue()
 547                                         + ":INPORT")))));
 548             return flowb.build();
 549         } else {
 550             LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId()
 551                 .getValue());
 552             return null;
 553         }
 554
 555     }
 556
 557     private Flow createLocalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) {
 558
 559         // TODO Li alagalah - refactor common code but keep simple method
 560         ArrayList<Instruction> instructions = new ArrayList<>();
 561         List<Action> applyActions = new ArrayList<>();
 562
 563         int order = 0;
 564
 565         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 566         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 567         Action setNextHop;
 568         String nextHop;
 569
 570         // BEGIN L2 LOCAL
 571         nextHop = ofc.getNodeConnectorId().getValue();
 572
 573         long portNum;
 574         try {
 575             portNum = getOfPortNum(ofc.getNodeConnectorId());
 576         } catch (NumberFormatException ex) {
 577             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 578             return null;
 579         }
 580
 581         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 582
 583         // END L2 LOCAL
 584
 585         order += 1;
 586         applyActions.add(setdEPG);
 587         applyActions.add(setdCG);
 588         applyActions.add(setNextHop);
 589         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 590             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 591             .build();
 592         instructions.add(applyActionsIns);
 593
 594         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 595             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 596             .build();
 597         instructions.add(gotoTable);
 598
 599         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 600         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
 601         Match match = mb.build();
 602         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL2", match);
 603         FlowBuilder flowb = base().setId(flowid)
 604             .setPriority(Integer.valueOf(50))
 605             .setMatch(match)
 606             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 607         return flowb.build();
 608     }
 609
 610     private void syncEP(FlowMap flowMap, NodeId nodeId, PolicyInfo policyInfo, Endpoint srcEp, Endpoint destEp)
 611             throws Exception {
 612
 613         if (ctx.getPolicyResolver().getTenant(srcEp.getTenant()) == null
 614                 || ctx.getPolicyResolver().getTenant(destEp.getTenant()) == null) {
 615             LOG.debug("Source or destination EP references empty tenant srcEp:{} destEp:{}", srcEp, destEp);
 616             return;
 617         }
 618
 619         // TODO: Conditions messed up, but for now, send policyInfo until this
 620         // is fixed.
 621         EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, destEp);
 622         if (destEpFwdCtxOrds == null) {
 623             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", destEp);
 624             return;
 625         }
 626         EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, srcEp);
 627         if (srcEpFwdCtxOrds == null) {
 628             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", srcEp);
 629             return;
 630         }
 631
 632
 633         if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) {
 634             if (destEp.getTenant() == null) {
 635                 LOG.debug("Didn't process endpoint {} due to tenant being null", destEp.getKey());
 636             } else {
 637                 LOG.debug("Didn't process endpoint {} due to EPG(s) being null", destEp.getKey());
 638             }
 639             return;
 640         }
 641         OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class);
 642
 643         if (LocationType.External.equals(ofc.getLocationType())) {
 644             LOG.error("syncEp(): External endpoints should not be seen here.");
 645             return;
 646         }
 647
 648         /*
 649          * Only care about subnets for L3, but fetch them before loop. We need
 650          * the local subnets for setting SRC MAC for routing. All Routing is now
 651          * done locally! YAY! Instead of being shovelled L2 style across network
 652          * ala Helium.
 653          */
 654         List<Subnet> localSubnets = getLocalSubnets(nodeId);
 655         if (localSubnets == null) {
 656             LOG.error("No subnets could be found locally for node: {}", nodeId);
 657             return;
 658         }
 659
 660         if (Objects.equals(ofc.getNodeId(), nodeId)) {
 661             // this is a local endpoint; send to the approppriate local
 662             // port
 663
 664             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 665                 flowMap.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc));
 666             }
 667             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 668             // The Endpoint conflation must end!
 669             if (destEp.getL3Address() == null) {
 670                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 671                 return;
 672             }
 673
 674             for (L3Address l3a : destEp.getL3Address()) {
 675                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 676                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 677                             destEp.getL3Address());
 678                     continue;
 679                 } else {
 680                     for (Subnet localSubnet : localSubnets) {
 681                         Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet);
 682                         if (flow != null) {
 683                             flowMap.writeFlow(nodeId, TABLE_ID, flow);
 684                         } else {
 685                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 686                                     localSubnet.getIpPrefix().getValue());
 687                         }
 688                     }
 689                 }
 690             }
 691         } else {
 692             // this endpoint is on a different switch; send to the
 693             // appropriate tunnel
 694             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 695                 Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc);
 696                 if (remoteL2Flow != null) {
 697                     flowMap.writeFlow(nodeId, TABLE_ID, remoteL2Flow);
 698                 }
 699             } else {
 700                 LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}",
 701                         srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId());
 702             }
 703
 704             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 705             // The Endpoint conflation must end!
 706             if (destEp.getL3Address() == null) {
 707                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 708                 return;
 709             }
 710             for (L3Address l3a : destEp.getL3Address()) {
 711                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 712                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 713                             destEp.getL3Address());
 714                     continue;
 715                 } else {
 716                     for (Subnet localSubnet : localSubnets) {
 717                         Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds,
 718                                 destEpFwdCtxOrds, ofc, localSubnet);
 719                         if (remoteL3Flow != null) {
 720                             flowMap.writeFlow(nodeId, TABLE_ID, remoteL3Flow);
 721                         } else {
 722                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 723                                     localSubnet.getIpPrefix().getValue());
 724                         }
 725                     }
 726                 }
 727             }
 728         } // remote (tunnel)
 729
 730         // }
 731
 732     }
 733
 734     /*
 735      * ################################## DestMapper Flow methods
 736      * ##################################
 737      */
 738     private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
 739             OfOverlayContext ofc, Subnet srcSubnet) {
 740
 741         // TODO Li alagalah - refactor common code but keep simple method
 742
 743         Subnet destSubnet = null;
 744         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 745         if (subnets == null) {
 746             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 747             return null;
 748         }
 749         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 750         for (Subnet subnet : subnets) {
 751             // TODO Li alagalah add IPv6 support
 752             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 753                 destSubnet = subnet;
 754                 break;
 755             }
 756         }
 757         if (destSubnet == null) {
 758             LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 759             return null;
 760         }
 761
 762         if (destSubnet.getVirtualRouterIp() == null) {
 763             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 764                     destL3Address.getKey());
 765             return null;
 766         }
 767
 768         if (srcSubnet.getVirtualRouterIp() == null) {
 769             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 770             return null;
 771         }
 772         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 773         if (destL3c == null || destL3c.getId() == null) {
 774             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 775             return null;
 776         }
 777         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 778         if (srcL3c == null || srcL3c.getId() == null) {
 779             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 780             return null;
 781         }
 782
 783         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 784             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 785                 .getValue(), destL3c.getId().getValue());
 786             return null;
 787         }
 788
 789         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 790         MacAddress epDestMac = destEp.getMacAddress();
 791         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 792
 793         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 794             // This is our final destination, so match on actual EP mac.
 795             matcherMac = epDestMac;
 796         }
 797
 798         ArrayList<Instruction> l3instructions = new ArrayList<>();
 799         List<Action> applyActions = new ArrayList<>();
 800         List<Action> l3ApplyActions = new ArrayList<>();
 801
 802         int order = 0;
 803
 804         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 805         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 806         Action setNextHop;
 807         String nextHop;
 808
 809         // BEGIN L3 LOCAL
 810         nextHop = ofc.getNodeConnectorId().getValue();
 811
 812         long portNum;
 813         try {
 814             portNum = getOfPortNum(ofc.getNodeConnectorId());
 815         } catch (NumberFormatException ex) {
 816             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 817             return null;
 818         }
 819
 820         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 821         // END L3 LOCAL
 822
 823         // Lets not re-write the srcMac if its local.
 824         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
 825             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
 826             l3ApplyActions.add(setDlSrc);
 827         }
 828
 829         Action setDlDst = setDlDstAction(epDestMac);
 830         l3ApplyActions.add(setDlDst);
 831
 832         Action decTtl = decNwTtlAction();
 833         l3ApplyActions.add(decTtl);
 834
 835         order += 1;
 836         applyActions.add(setdEPG);
 837         applyActions.add(setdCG);
 838         applyActions.add(setNextHop);
 839
 840         applyActions.addAll(l3ApplyActions);
 841         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 842             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 843             .build();
 844
 845         l3instructions.add(applyActionsIns);
 846         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 847             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 848             .build();
 849         l3instructions.add(gotoTable);
 850         Layer3Match m = null;
 851         Long etherType = null;
 852         String ikey = null;
 853         if (destL3Address.getIpAddress().getIpv4Address() != null) {
 854             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
 855             etherType = IPv4;
 856             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 857         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
 858             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
 859             etherType = IPv6;
 860             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 861         } else {
 862             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
 863             return null;
 864         }
 865
 866         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
 867             .setLayer3Match(m);
 868         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 869         Match match = mb.build();
 870         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL3", match);
 871         FlowBuilder flowb = base().setId(flowid)
 872             .setPriority(Integer.valueOf(132))
 873             .setMatch(match)
 874             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 875         return flowb.build();
 876     }
 877
 878     private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds,
 879             EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) {
 880
 881         // TODO Li alagalah - refactor common code but keep simple method
 882
 883         // this endpoint is on a different switch; send to the
 884         // appropriate tunnel
 885
 886         ArrayList<Instruction> instructions = new ArrayList<>();
 887         List<Action> applyActions = new ArrayList<>();
 888
 889         int order = 0;
 890
 891         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
 892         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
 893         Action setNextHop;
 894         String nextHop;
 895
 896         // BEGIN TUNNEL HANDLING
 897         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
 898         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
 899         if (tunDst == null) {
 900             LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep);
 901             return null;
 902         }
 903         if (tunPort == null) {
 904             LOG.warn("Failed to get Tunnel Port for NodeId {} with EP {}", nodeId, ep);
 905             return null;
 906         }
 907
 908         Action tundstAction;
 909
 910         if (tunDst.getIpv4Address() != null) {
 911             nextHop = tunDst.getIpv4Address().getValue();
 912             tundstAction = nxLoadTunIPv4Action(nextHop, false);
 913         } else if (tunDst.getIpv6Address() != null) {
 914             // nextHop = tunDst.getIpv6Address().getValue();
 915             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
 916                     ofc.getNodeId());
 917             return null;
 918         } else {
 919             // this shouldn't happen
 920             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
 921             return null;
 922         }
 923
 924         long portNum;
 925         try {
 926             portNum = getOfPortNum(tunPort);
 927         } catch (NumberFormatException ex) {
 928             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 929             return null;
 930         }
 931
 932         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 933         applyActions.add(tundstAction);
 934         // END TUNNEL
 935
 936         order += 1;
 937         applyActions.add(setdEPG);
 938         applyActions.add(setdCG);
 939         applyActions.add(setNextHop);
 940         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 941             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 942             .build();
 943         instructions.add(applyActionsIns);
 944
 945         applyActionsIns = new InstructionBuilder().setOrder(order++)
 946             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 947             .build();
 948
 949         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 950             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 951             .build();
 952         instructions.add(gotoTable);
 953
 954         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 955         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId())));
 956         Match match = mb.build();
 957         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL2", match);
 958         FlowBuilder flowb = base().setId(flowid)
 959             .setPriority(Integer.valueOf(50))
 960             .setMatch(match)
 961             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 962
 963         return flowb.build();
 964     }
 965
 966     private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId,
 967             EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc,
 968             Subnet srcSubnet) {
 969
 970         // TODO Li alagalah - refactor common code but keep simple method
 971
 972         // this endpoint is on a different switch; send to the
 973         // appropriate tunnel
 974         Subnet destSubnet = null;
 975         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 976         if (subnets == null) {
 977             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 978             return null;
 979         }
 980         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 981         for (Subnet subnet : subnets) {
 982             // TODO Li alagalah add IPv6 support
 983             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 984                 destSubnet = subnet;
 985                 break;
 986             }
 987         }
 988         if (destSubnet == null) {
 989             LOG.info("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 990             return null;
 991         }
 992
 993         if (destSubnet.getVirtualRouterIp() == null) {
 994             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 995                     destL3Address.getKey());
 996             return null;
 997         }
 998
 999         if (srcSubnet.getVirtualRouterIp() == null) {
1000             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
1001             return null;
1002         }
1003         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
1004         if (destL3c == null || destL3c.getId() == null) {
1005             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
1006             return null;
1007         }
1008         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
1009         if (srcL3c == null || srcL3c.getId() == null) {
1010             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
1011             return null;
1012         }
1013
1014         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
1015             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
1016                 .getValue(), destL3c.getId().getValue());
1017             return null;
1018         }
1019
1020         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
1021         MacAddress epDestMac = destEp.getMacAddress();
1022         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
1023         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
1024             // This is our final destination, so match on actual EP mac.
1025             matcherMac = epDestMac;
1026         }
1027         ArrayList<Instruction> l3instructions = new ArrayList<>();
1028         List<Action> applyActions = new ArrayList<>();
1029         List<Action> l3ApplyActions = new ArrayList<>();
1030
1031         int order = 0;
1032
1033         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
1034         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
1035         Action setNextHop;
1036         String nextHop;
1037
1038         // BEGIN TUNNEL HANDLING
1039         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
1040         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
1041         if (tunDst == null) {
1042             LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address);
1043             return null;
1044         }
1045         if (tunPort == null) {
1046             LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, destL3Address);
1047             return null;
1048         }
1049
1050         Action tundstAction;
1051
1052         if (tunDst.getIpv4Address() != null) {
1053             nextHop = tunDst.getIpv4Address().getValue();
1054             tundstAction = nxLoadTunIPv4Action(nextHop, false);
1055         } else if (tunDst.getIpv6Address() != null) {
1056             // nextHop = tunDst.getIpv6Address().getValue();
1057             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
1058                     ofc.getNodeId());
1059             return null;
1060         } else {
1061             // this shouldn't happen
1062             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
1063             return null;
1064         }
1065
1066         long portNum;
1067         try {
1068             portNum = getOfPortNum(tunPort);
1069         } catch (NumberFormatException ex) {
1070             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
1071             return null;
1072         }
1073
1074         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1075         applyActions.add(tundstAction);
1076         // END TUNNEL
1077
1078         order += 1;
1079         applyActions.add(setdEPG);
1080         applyActions.add(setdCG);
1081         applyActions.add(setNextHop);
1082
1083         // Lets not re-write the srcMac if its local.
1084         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
1085             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
1086             l3ApplyActions.add(setDlSrc);
1087         }
1088
1089         Action setDlDst = setDlDstAction(epDestMac);
1090         l3ApplyActions.add(setDlDst);
1091
1092         Action decTtl = decNwTtlAction();
1093         l3ApplyActions.add(decTtl);
1094
1095         applyActions.addAll(l3ApplyActions);
1096         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1097             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1098             .build();
1099
1100         l3instructions.add(applyActionsIns);
1101         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1102             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1103             .build();
1104         l3instructions.add(gotoTable);
1105         Layer3Match m = null;
1106         Long etherType = null;
1107         String ikey = null;
1108         if (destL3Address.getIpAddress().getIpv4Address() != null) {
1109             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
1110             etherType = IPv4;
1111             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
1112         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
1113             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
1114             etherType = IPv6;
1115             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
1116         } else {
1117             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
1118             return null;
1119         }
1120
1121         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
1122             .setLayer3Match(m);
1123         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id())));
1124         Match match = mb.build();
1125         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL3", match);
1126         FlowBuilder flowb = base().setId(flowid)
1127             .setPriority(Integer.valueOf(132))
1128             .setMatch(match)
1129             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
1130         return flowb.build();
1131     }
1132
1133     private NetworkDomainId getEPNetworkContainment(Endpoint endpoint) {
1134         if (endpoint.getNetworkContainment() != null) {
1135             return endpoint.getNetworkContainment();
1136         } else {
1137             /*
1138              * TODO: Be alagalah: Endpoint Refactor: This should be set on input
1139              * which we can't do because of the backwards way endpoints were
1140              * "architected".
1141              */
1142             return ctx.getPolicyResolver()
1143                 .getTenant(endpoint.getTenant())
1144                 .getEndpointGroup(endpoint.getEndpointGroup())
1145                 .getNetworkDomain();
1146         }
1147     }
1148
1149     private HashSet<Subnet> getSubnets(final TenantId tenantId) {
1150
1151         // if (subnetsByTenant.get(tenantId) != null) {
1152         // return subnetsByTenant.get(tenantId);
1153         // }
1154
1155         if (ctx.getDataBroker() == null) {
1156             return null;
1157         }
1158
1159         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
1160         InstanceIdentifier<Tenant> tiid = TenantUtils.tenantIid(tenantId);
1161         Optional<Tenant> tenantInfo;
1162         try {
1163             tenantInfo = t.read(LogicalDatastoreType.CONFIGURATION, tiid).get();
1164         } catch (Exception e) {
1165             LOG.error("Could not read Tenant {}", tenantId, e);
1166             return null;
1167         }
1168
1169         HashSet<Subnet> subnets = new HashSet<Subnet>();
1170
1171         if (!tenantInfo.isPresent()) {
1172             LOG.warn("Tenant {} not found", tenantId);
1173             return null;
1174         }
1175
1176         subnets.addAll(tenantInfo.get().getSubnet());
1177         // subnetsByTenant.put(tenantId, subnets);
1178         return subnets;
1179     }
1180
1181     // Need a method to get subnets for EPs attached to the node locally
1182     // to set the source Mac address for the router interface.
1183     private List<Subnet> getLocalSubnets(NodeId nodeId) {
1184         Collection<Endpoint> endpointsForNode = ctx.getEndpointManager().getEndpointsForNode(nodeId);
1185
1186         List<Subnet> localSubnets = new ArrayList<Subnet>();
1187
1188         for (Endpoint endpoint : endpointsForNode) {
1189             HashSet<Subnet> subnets = getSubnets(endpoint.getTenant());
1190             if (subnets == null) {
1191                 LOG.debug("No local subnets in tenant {} for EP {}.", endpoint.getTenant(), endpoint.getKey());
1192                 continue;
1193             }
1194             NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint);
1195             for (Subnet subnet : subnets) {
1196                 if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) {
1197                     localSubnets.add(subnet);
1198                 }
1199             }
1200         }
1201         return localSubnets;
1202     }
1203
1204     static byte[] bytesFromHexString(String values) {
1205         String target = "";
1206         if (values != null) {
1207             target = values;
1208         }
1209         String[] octets = target.split(":");
1210
1211         byte[] ret = new byte[octets.length];
1212         for (int i = 0; i < octets.length; i++) {
1213             ret[i] = Integer.valueOf(octets[i], 16).byteValue();
1214         }
1215         return ret;
1216     }
1217 }