renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java

   1 /*
   2  * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow;
  10
  11 import static com.google.common.base.Preconditions.checkNotNull;
  12 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP;
  13 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4;
  14 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6;
  15 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch;
  16 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns;
  17 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction;
  18 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch;
  19 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum;
  20 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.gotoTableIns;
  21 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.groupAction;
  22 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.instructions;
  23 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpOpAction;
  24 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpShaAction;
  25 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction;
  26 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction;
  27 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action;
  28 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction;
  29 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction;
  30 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction;
  31 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction;
  32 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction;
  33 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction;
  34 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction;
  35 import static org.opendaylight.groupbasedpolicy.util.DataStoreHelper.readFromDs;
  36
  37 import java.math.BigInteger;
  38 import java.util.ArrayList;
  39 import java.util.Collection;
  40 import java.util.Collections;
  41 import java.util.HashMap;
  42 import java.util.HashSet;
  43 import java.util.List;
  44 import java.util.Map;
  45 import java.util.Map.Entry;
  46 import java.util.Objects;
  47 import java.util.Set;
  48
  49 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  50 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  51 import org.opendaylight.groupbasedpolicy.dto.EgKey;
  52 import org.opendaylight.groupbasedpolicy.dto.EpKey;
  53 import org.opendaylight.groupbasedpolicy.dto.IndexedTenant;
  54 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext;
  55 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfWriter;
  56 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch;
  57 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals;
  58 import org.opendaylight.groupbasedpolicy.util.IidFactory;
  59 import org.opendaylight.groupbasedpolicy.util.TenantUtils;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix;
  63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.l3.prefix.fields.EndpointL3Gateways;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Prefix;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.EndpointLocation.LocationType;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.ForwardingContext;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L3Context;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.Subnet;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.ethernet.match.fields.EthernetDestinationBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatchBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.Layer3Match;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 104 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 105 import org.slf4j.Logger;
 106 import org.slf4j.LoggerFactory;
 107
 108 import com.google.common.base.Optional;
 109 import com.google.common.base.Strings;
 110 import com.google.common.collect.HashMultimap;
 111 import com.google.common.collect.SetMultimap;
 112 import com.google.common.collect.Sets;
 113
 114 /**
 115  * Manage the table that maps the destination address to the next hop for the
 116  * path as well as applies any relevant routing transformations.
 117  */
 118 public class DestinationMapper extends FlowTable {
 119
 120     protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class);
 121
 122     // TODO Li alagalah: Improve UT coverage for this class.
 123
 124     // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows
 125     // This ensures we have the appropriate network-containment'
 126
 127     public static short TABLE_ID;
 128     /**
 129      * This is the MAC address of the magical router in the sky
 130      */
 131     public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5");
 132     public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00");
 133     public static final Integer BASE_L3_PRIORITY = 100;
 134
 135     public DestinationMapper(OfContext ctx, short tableId) {
 136         super(ctx);
 137         this.TABLE_ID = tableId;
 138     }
 139
 140     Map<TenantId, HashSet<Subnet>> subnetsByTenant = new HashMap<TenantId, HashSet<Subnet>>();
 141
 142     @Override
 143     public short getTableId() {
 144         return TABLE_ID;
 145     }
 146
 147     @Override
 148     public void sync(NodeId nodeId, OfWriter ofWriter) throws Exception {
 149
 150         TenantId currentTenant = null;
 151
 152         ofWriter.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null, TABLE_ID));
 153
 154         SetMultimap<EpKey, EpKey> visitedEps = HashMultimap.create();
 155         Set<EndpointFwdCtxOrdinals> epOrdSet = new HashSet<>();
 156
 157         for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) {
 158             Set<EndpointGroupId> srcEpgIds = new HashSet<>();
 159             if (srcEp.getEndpointGroup() != null)
 160                 srcEpgIds.add(srcEp.getEndpointGroup());
 161             if (srcEp.getEndpointGroups() != null)
 162                 srcEpgIds.addAll(srcEp.getEndpointGroups());
 163
 164             for (EndpointGroupId epgId : srcEpgIds) {
 165                 EgKey epg = new EgKey(srcEp.getTenant(), epgId);
 166                 Set<EgKey> peers = Sets.union(Collections.singleton(epg), ctx.getCurrentPolicy().getPeers(epg));
 167                 for (EgKey peer : peers) {
 168                     for (Endpoint peerEp : ctx.getEndpointManager().getEndpointsForGroup(peer)) {
 169                         currentTenant = peerEp.getTenant();
 170                         subnetsByTenant.put(currentTenant, getSubnets(currentTenant));
 171                         EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress());
 172                         EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress());
 173
 174                         if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) {
 175                             continue;
 176                         }
 177                         syncEP(ofWriter, nodeId, srcEp, peerEp);
 178                         visitedEps.put(srcEpKey, peerEpKey);
 179
 180                         // Process subnets and flood-domains for epPeer
 181                         EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx,
 182                                 peerEp);
 183                         if (epOrds == null) {
 184                             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", peerEp);
 185                             continue;
 186                         }
 187
 188                         epOrdSet.add(epOrds);
 189                     }
 190                 }
 191             }
 192         }
 193
 194         for (Entry<TenantId, HashSet<Subnet>> subnetEntry : subnetsByTenant.entrySet()) {
 195             if (subnetEntry.getValue() == null) {
 196                 LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey());
 197                 continue;
 198             }
 199             currentTenant = subnetEntry.getKey();
 200             for (Subnet sn : subnetEntry.getValue()) {
 201                 L3Context l3c = getL3ContextForSubnet(currentTenant, sn);
 202                 Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn,
 203                         OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId()));
 204                 if (arpFlow != null) {
 205                     ofWriter.writeFlow(nodeId, TABLE_ID, arpFlow);
 206                 } else {
 207                     LOG.debug(
 208                             "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .",
 209                             sn.getIpPrefix().getValue());
 210                 }
 211             }
 212         }
 213
 214         // Write broadcast flows per flood domain.
 215         for (EndpointFwdCtxOrdinals epOrd : epOrdSet) {
 216             if (ofWriter.groupExists(nodeId, Integer.valueOf(epOrd.getFdId()).longValue())) {
 217                 ofWriter.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd));
 218             }
 219         }
 220
 221         // L3 Prefix Endpoint handling
 222         Collection<EndpointL3Prefix> prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(currentTenant);
 223         if (prefixEps != null) {
 224             LOG.trace("DestinationMapper - Processing L3PrefixEndpoints");
 225             for (EndpointL3Prefix prefixEp : prefixEps) {
 226                 List<Subnet> localSubnets = getLocalSubnets(nodeId);
 227                 if (localSubnets == null) {
 228                     continue;
 229                 }
 230                 for (Subnet localSubnet: localSubnets) {
 231                     Flow prefixFlow = createL3PrefixFlow(prefixEp, nodeId, localSubnet);
 232                     if (prefixFlow != null) {
 233                         ofWriter.writeFlow(nodeId, TABLE_ID, prefixFlow);
 234                         LOG.trace("Wrote L3Prefix flow");
 235                     }
 236                 }
 237             }
 238         }
 239     }
 240
 241     // set up next-hop destinations for all the endpoints in the endpoint
 242     // group on the node
 243
 244     private Flow createL3PrefixFlow(EndpointL3Prefix prefixEp, NodeId nodeId, Subnet subnet) throws Exception {
 245         /*
 246          * Priority: 100+lengthprefix
 247          * Match: prefix, l3c, "mac address of router" ?
 248          * Action:
 249          * - set Reg2, Reg3 for L3Ep by L2Ep ?
 250          * - if external,
 251          * - Reg7: use switch location external port else punt for now
 252          * - if internal
 253          * - Reg7: grab L2Ep from L3Ep and use its location info
 254          * - goto_table: POLENF (will check there for external on EP)
 255          */
 256
 257         ReadOnlyTransaction rTx = ctx.getDataBroker().newReadOnlyTransaction();
 258         // TODO Bug #3440 Target: Be - should support for more than first gateway.
 259         EndpointL3Gateways l3Gateway = prefixEp.getEndpointL3Gateways().get(0);
 260         Optional<EndpointL3> optL3Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 261                 IidFactory.l3EndpointIid(l3Gateway.getL3Context(), l3Gateway.getIpAddress()), rTx);
 262         if (!optL3Ep.isPresent()) {
 263             LOG.error("createL3PrefixFlow - L3Endpoint gateway {} for L3Prefix {} not found.", l3Gateway, prefixEp);
 264             return null;
 265         }
 266         EndpointL3 l3Ep = optL3Ep.get();
 267         if (l3Ep.getL2Context() == null || l3Ep.getMacAddress() == null) {
 268             LOG.debug("L3 endpoint representing L3 gateway does not contain L2-context or MAC address. {}", l3Ep);
 269             return null;
 270         }
 271         Optional<Endpoint> optL2Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 272                 IidFactory.endpointIid(l3Ep.getL2Context(), l3Ep.getMacAddress()), rTx);
 273         if (!optL2Ep.isPresent()) {
 274             LOG.error("createL3PrefixFlow - L2Endpoint for L3Gateway {} not found.", l3Ep);
 275             return null;
 276         }
 277         Endpoint l2Ep = optL2Ep.get();
 278         EndpointFwdCtxOrdinals epFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, l2Ep);
 279         if (epFwdCtxOrds == null) {
 280             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", l2Ep);
 281             return null;
 282         }
 283
 284         NetworkDomainId epNetworkContainment = getEPNetworkContainment(l2Ep);
 285
 286         MacAddress epDestMac = l2Ep.getMacAddress();
 287         MacAddress destSubnetGatewayMac = l2Ep.getMacAddress();
 288         L3Context destL3c = getL3ContextForSubnet(prefixEp.getTenant(), subnet);
 289         if (destL3c == null || destL3c.getId() == null) {
 290             LOG.error("No L3 Context found associated with subnet {}", subnet.getId());
 291             return null;
 292         }
 293
 294         MacAddress matcherMac = routerPortMac(destL3c, subnet.getVirtualRouterIp());
 295
 296         ArrayList<Instruction> l3instructions = new ArrayList<>();
 297         List<Action> applyActions = new ArrayList<>();
 298         List<Action> l3ApplyActions = new ArrayList<>();
 299
 300         int order = 0;
 301
 302         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 303         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 304         Action setNextHop;
 305         String nextHop=null;
 306
 307         OfOverlayContext ofc = l2Ep.getAugmentation(OfOverlayContext.class);
 308         LocationType location;
 309
 310         if (ofc != null && ofc.getLocationType() != null) {
 311             location = ofc.getLocationType();
 312         } else if (ofc != null) {
 313             // Augmentation, but using default location
 314             location = LocationType.Internal;
 315         } else {
 316             LOG.info("createL3PrefixFlow - Endpoint {} had no augmentation.", l2Ep);
 317             return null;
 318         }
 319
 320         long portNum = -1;
 321
 322         if (location.equals(LocationType.Internal)) {
 323             checkNotNull(ofc.getNodeConnectorId());
 324             nextHop = ofc.getNodeConnectorId().getValue();
 325             try {
 326                 portNum = getOfPortNum(ofc.getNodeConnectorId());
 327             } catch (NumberFormatException ex) {
 328                 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 329                 return null;
 330             }
 331
 332         } else {
 333             // External
 334             Set<NodeConnectorId> externalPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
 335             checkNotNull(externalPorts);
 336             for (NodeConnectorId externalPort : externalPorts) {
 337                 // TODO Bug #3440 Target: Be - should support for more than first external port.
 338                 //TODO Bug 3546 - Difficult: External port is unrelated to Tenant, L3C, L2BD..
 339                 nextHop = externalPort.getValue();
 340                 try {
 341                     portNum = getOfPortNum(externalPort);
 342                 } catch (NumberFormatException ex) {
 343                     LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 344                     return null;
 345                 }
 346                 continue;
 347             }
 348         }
 349
 350         if (Strings.isNullOrEmpty(nextHop)
 351                 || portNum == -1) {
 352             LOG.error("createL3Prefix - Cannot find nodeConnectorId for {} for Prefix: ", l2Ep, prefixEp);
 353             return null;
 354         }
 355         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 356
 357         Action setDlDst = setDlDstAction(epDestMac);
 358         l3ApplyActions.add(setDlDst);
 359
 360         Action decTtl = decNwTtlAction();
 361         l3ApplyActions.add(decTtl);
 362
 363         order += 1;
 364         applyActions.add(setdEPG);
 365         applyActions.add(setdCG);
 366         applyActions.add(setNextHop);
 367
 368         applyActions.addAll(l3ApplyActions);
 369         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 370             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 371             .build();
 372
 373         l3instructions.add(applyActionsIns);
 374         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 375             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 376             .build();
 377         l3instructions.add(gotoTable);
 378
 379         Layer3Match m = null;
 380         Long etherType = null;
 381         String ikey = null;
 382         Integer prefixLength=0;
 383         if (prefixEp.getIpPrefix().getIpv4Prefix() != null) {
 384             ikey = prefixEp.getIpPrefix().getIpv4Prefix().getValue();
 385             etherType = IPv4;
 386             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv4Prefix().getValue().split("/")[1]);
 387             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 388         } else if (prefixEp.getIpPrefix().getIpv6Prefix() != null) {
 389             ikey = prefixEp.getIpPrefix().getIpv6Prefix().getValue();
 390             etherType = IPv6;
 391             /*
 392              *  This will result in flows with priority between 100-228, but since its matching on IPv6 prefix as well
 393              *  this shouldn't pose and issue, as the priority is more important within the address space of the matcher,
 394              *  even though technically flows are processed in priority order.
 395              */
 396
 397             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv6Prefix().getValue().split("/")[1]);
 398             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 399         } else {
 400             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", prefixEp);
 401             return null;
 402         }
 403
 404         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType));
 405         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 406         Match match = mb.build();
 407         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "L3prefix", match);
 408         FlowBuilder flowb = base().setId(flowid)
 409             .setPriority(Integer.valueOf(BASE_L3_PRIORITY+prefixLength))
 410             .setMatch(match)
 411             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 412         return flowb.build();
 413     }
 414
 415     private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) {
 416         MatchBuilder mb = new MatchBuilder()
 417                 .setEthernetMatch(new EthernetMatchBuilder().setEthernetDestination(
 418                         new EthernetDestinationBuilder().setAddress(MULTICAST_MAC)
 419                                 .setMask(MULTICAST_MAC)
 420                                 .build()).build());
 421         addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId())));
 422
 423         Match match = mb.build();
 424         FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "broadcast", match);
 425         FlowBuilder flowb = base().setPriority(Integer.valueOf(140))
 426             .setId(flowId)
 427             .setMatch(match)
 428             .setInstructions(
 429                     instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false),
 430                             groupAction(Long.valueOf(epOrd.getFdId())))));
 431
 432         return flowb.build();
 433     }
 434
 435     private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) {
 436
 437         if (ctx.getDataBroker() == null) {
 438             return null;
 439         }
 440
 441         MacAddress defaultMacAddress = ROUTER_MAC;
 442
 443         EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId());
 444         InstanceIdentifier<EndpointL3> endpointsIid = InstanceIdentifier.builder(Endpoints.class)
 445             .child(EndpointL3.class, l3Key)
 446             .build();
 447         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 448
 449         Optional<EndpointL3> r;
 450         try {
 451             r = t.read(LogicalDatastoreType.OPERATIONAL, endpointsIid).get();
 452             if (!r.isPresent())
 453                 return defaultMacAddress;
 454             EndpointL3 epL3 = r.get();
 455             if (epL3.getMacAddress() == null) {
 456                 return defaultMacAddress;
 457             } else {
 458                 return epL3.getMacAddress();
 459             }
 460         } catch (Exception e) {
 461             LOG.error("Error reading EndpointL3 {}.{}", l3c, ipAddress, e);
 462             return null;
 463         }
 464     }
 465
 466     private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) {
 467         IndexedTenant indexedTenant = ctx.getTenant(tenantId);
 468         if (indexedTenant == null) {
 469             LOG.debug("Tenant {} is null, cannot get L3 context", tenantId);
 470             return null;
 471         }
 472         L3Context l3c = indexedTenant.resolveL3Context(sn.getId());
 473         return l3c;
 474     }
 475
 476     private Flow createRouterArpFlow(TenantId tenantId, NodeId nodeId, Subnet sn, int l3Id) {
 477         if (sn == null || sn.getVirtualRouterIp() == null) {
 478             LOG.trace("Didn't create routerArpFlow since either subnet or subnet virtual router was null");
 479             return null;
 480         }
 481         /*
 482          * TODO: Li alagalah: This should be new Yang "gateways" list as well,
 483          * that expresses the gateway and prefixes it is interface for. Should
 484          * also check for external.
 485          */
 486         if (sn.getVirtualRouterIp().getIpv4Address() != null) {
 487             String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue();
 488
 489             L3Context l3c = getL3ContextForSubnet(tenantId, sn);
 490             if (l3c == null) {
 491                 LOG.error("No L3 Context found associated with subnet {}", sn.getId());
 492             }
 493
 494             MacAddress routerMac = routerPortMac(l3c, sn.getVirtualRouterIp());
 495             if (routerMac == null) {
 496                 return null;
 497             }
 498
 499             BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue()));
 500
 501             MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match(
 502                     new ArpMatchBuilder().setArpOp(Integer.valueOf(1))
 503                         .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32"))
 504                         .build());
 505             addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id)));
 506
 507             Match match = mb.build();
 508             FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "routerarp", match);
 509             FlowBuilder flowb = base().setPriority(150)
 510                 .setId(flowId)
 511                 .setMatch(match)
 512                 .setInstructions(
 513                         instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac),
 514                                 nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(),
 515                                 nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(),
 516                                 nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue()
 517                                         + ":INPORT")))));
 518             return flowb.build();
 519         } else {
 520             LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId()
 521                 .getValue());
 522             return null;
 523         }
 524
 525     }
 526
 527     private Flow createLocalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) {
 528
 529         // TODO Li alagalah - refactor common code but keep simple method
 530         ArrayList<Instruction> instructions = new ArrayList<>();
 531         List<Action> applyActions = new ArrayList<>();
 532
 533         int order = 0;
 534
 535         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 536         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 537         Action setNextHop;
 538         String nextHop;
 539
 540         // BEGIN L2 LOCAL
 541         nextHop = ofc.getNodeConnectorId().getValue();
 542
 543         long portNum;
 544         try {
 545             portNum = getOfPortNum(ofc.getNodeConnectorId());
 546         } catch (NumberFormatException ex) {
 547             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 548             return null;
 549         }
 550
 551         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 552
 553         // END L2 LOCAL
 554
 555         order += 1;
 556         applyActions.add(setdEPG);
 557         applyActions.add(setdCG);
 558         applyActions.add(setNextHop);
 559         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 560             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 561             .build();
 562         instructions.add(applyActionsIns);
 563
 564         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 565             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 566             .build();
 567         instructions.add(gotoTable);
 568
 569         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 570         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
 571         Match match = mb.build();
 572         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL2", match);
 573         FlowBuilder flowb = base().setId(flowid)
 574             .setPriority(Integer.valueOf(50))
 575             .setMatch(match)
 576             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 577         return flowb.build();
 578     }
 579
 580     private void syncEP(OfWriter ofWriter, NodeId nodeId, Endpoint srcEp, Endpoint destEp)
 581             throws Exception {
 582
 583         if (ctx.getTenant(srcEp.getTenant()) == null
 584                 || ctx.getTenant(destEp.getTenant()) == null) {
 585             LOG.debug("Source or destination EP references empty tenant srcEp:{} destEp:{}", srcEp, destEp);
 586             return;
 587         }
 588
 589         // TODO: Conditions messed up, but for now, send policyInfo until this
 590         // is fixed.
 591         EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, destEp);
 592         if (destEpFwdCtxOrds == null) {
 593             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", destEp);
 594             return;
 595         }
 596         EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, srcEp);
 597         if (srcEpFwdCtxOrds == null) {
 598             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", srcEp);
 599             return;
 600         }
 601
 602
 603         if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) {
 604             if (destEp.getTenant() == null) {
 605                 LOG.debug("Didn't process endpoint {} due to tenant being null", destEp.getKey());
 606             } else {
 607                 LOG.debug("Didn't process endpoint {} due to EPG(s) being null", destEp.getKey());
 608             }
 609             return;
 610         }
 611         OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class);
 612
 613         if (LocationType.External.equals(ofc.getLocationType())) {
 614             LOG.error("syncEp(): External endpoints should not be seen here.");
 615             return;
 616         }
 617
 618         /*
 619          * Only care about subnets for L3, but fetch them before loop. We need
 620          * the local subnets for setting SRC MAC for routing. All Routing is now
 621          * done locally! YAY! Instead of being shovelled L2 style across network
 622          * ala Helium.
 623          */
 624         List<Subnet> localSubnets = getLocalSubnets(nodeId);
 625         if (localSubnets == null) {
 626             LOG.error("No subnets could be found locally for node: {}", nodeId);
 627             return;
 628         }
 629
 630         if (Objects.equals(ofc.getNodeId(), nodeId)) {
 631             // this is a local endpoint; send to the approppriate local
 632             // port
 633
 634             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 635                 ofWriter.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc));
 636             }
 637             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 638             // The Endpoint conflation must end!
 639             if (destEp.getL3Address() == null) {
 640                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 641                 return;
 642             }
 643
 644             for (L3Address l3a : destEp.getL3Address()) {
 645                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 646                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 647                             destEp.getL3Address());
 648                     continue;
 649                 } else {
 650                     for (Subnet localSubnet : localSubnets) {
 651                         Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet);
 652                         if (flow != null) {
 653                             ofWriter.writeFlow(nodeId, TABLE_ID, flow);
 654                         } else {
 655                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 656                                     localSubnet.getIpPrefix().getValue());
 657                         }
 658                     }
 659                 }
 660             }
 661         } else {
 662             // this endpoint is on a different switch; send to the
 663             // appropriate tunnel
 664             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 665                 Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc);
 666                 if (remoteL2Flow != null) {
 667                     ofWriter.writeFlow(nodeId, TABLE_ID, remoteL2Flow);
 668                 }
 669             } else {
 670                 LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}",
 671                         srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId());
 672             }
 673
 674             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 675             // The Endpoint conflation must end!
 676             if (destEp.getL3Address() == null) {
 677                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 678                 return;
 679             }
 680             for (L3Address l3a : destEp.getL3Address()) {
 681                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 682                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 683                             destEp.getL3Address());
 684                     continue;
 685                 } else {
 686                     for (Subnet localSubnet : localSubnets) {
 687                         Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds,
 688                                 destEpFwdCtxOrds, ofc, localSubnet);
 689                         if (remoteL3Flow != null) {
 690                             ofWriter.writeFlow(nodeId, TABLE_ID, remoteL3Flow);
 691                         } else {
 692                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 693                                     localSubnet.getIpPrefix().getValue());
 694                         }
 695                     }
 696                 }
 697             }
 698         } // remote (tunnel)
 699
 700         // }
 701
 702     }
 703
 704     /*
 705      * ################################## DestMapper Flow methods
 706      * ##################################
 707      */
 708     private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
 709             OfOverlayContext ofc, Subnet srcSubnet) {
 710
 711         // TODO Li alagalah - refactor common code but keep simple method
 712
 713         Subnet destSubnet = null;
 714         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 715         if (subnets == null) {
 716             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 717             return null;
 718         }
 719         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 720         for (Subnet subnet : subnets) {
 721             // TODO Li alagalah add IPv6 support
 722             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 723                 destSubnet = subnet;
 724                 break;
 725             }
 726         }
 727         if (destSubnet == null) {
 728             LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 729             return null;
 730         }
 731
 732         if (destSubnet.getVirtualRouterIp() == null) {
 733             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 734                     destL3Address.getKey());
 735             return null;
 736         }
 737
 738         if (srcSubnet.getVirtualRouterIp() == null) {
 739             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 740             return null;
 741         }
 742         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 743         if (destL3c == null || destL3c.getId() == null) {
 744             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 745             return null;
 746         }
 747         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 748         if (srcL3c == null || srcL3c.getId() == null) {
 749             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 750             return null;
 751         }
 752
 753         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 754             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 755                 .getValue(), destL3c.getId().getValue());
 756             return null;
 757         }
 758
 759         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 760         MacAddress epDestMac = destEp.getMacAddress();
 761         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 762
 763         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 764             // This is our final destination, so match on actual EP mac.
 765             matcherMac = epDestMac;
 766         }
 767
 768         ArrayList<Instruction> l3instructions = new ArrayList<>();
 769         List<Action> applyActions = new ArrayList<>();
 770         List<Action> l3ApplyActions = new ArrayList<>();
 771
 772         int order = 0;
 773
 774         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 775         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 776         Action setNextHop;
 777         String nextHop;
 778
 779         // BEGIN L3 LOCAL
 780         nextHop = ofc.getNodeConnectorId().getValue();
 781
 782         long portNum;
 783         try {
 784             portNum = getOfPortNum(ofc.getNodeConnectorId());
 785         } catch (NumberFormatException ex) {
 786             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 787             return null;
 788         }
 789
 790         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 791         // END L3 LOCAL
 792
 793         // Lets not re-write the srcMac if its local.
 794         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
 795             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
 796             l3ApplyActions.add(setDlSrc);
 797         }
 798
 799         Action setDlDst = setDlDstAction(epDestMac);
 800         l3ApplyActions.add(setDlDst);
 801
 802         Action decTtl = decNwTtlAction();
 803         l3ApplyActions.add(decTtl);
 804
 805         order += 1;
 806         applyActions.add(setdEPG);
 807         applyActions.add(setdCG);
 808         applyActions.add(setNextHop);
 809
 810         applyActions.addAll(l3ApplyActions);
 811         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 812             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 813             .build();
 814
 815         l3instructions.add(applyActionsIns);
 816         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 817             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 818             .build();
 819         l3instructions.add(gotoTable);
 820         Layer3Match m = null;
 821         Long etherType = null;
 822         String ikey = null;
 823         if (destL3Address.getIpAddress().getIpv4Address() != null) {
 824             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
 825             etherType = IPv4;
 826             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 827         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
 828             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
 829             etherType = IPv6;
 830             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 831         } else {
 832             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
 833             return null;
 834         }
 835
 836         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
 837             .setLayer3Match(m);
 838         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 839         Match match = mb.build();
 840         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL3", match);
 841         FlowBuilder flowb = base().setId(flowid)
 842             .setPriority(Integer.valueOf(132))
 843             .setMatch(match)
 844             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 845         return flowb.build();
 846     }
 847
 848     private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds,
 849             EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) {
 850
 851         // TODO Li alagalah - refactor common code but keep simple method
 852
 853         // this endpoint is on a different switch; send to the
 854         // appropriate tunnel
 855
 856         ArrayList<Instruction> instructions = new ArrayList<>();
 857         List<Action> applyActions = new ArrayList<>();
 858
 859         int order = 0;
 860
 861         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
 862         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
 863         Action setNextHop;
 864         String nextHop;
 865
 866         // BEGIN TUNNEL HANDLING
 867         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
 868         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
 869         if (tunDst == null) {
 870             LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep);
 871             return null;
 872         }
 873         if (tunPort == null) {
 874             LOG.warn("Failed to get Tunnel Port for NodeId {} with EP {}", nodeId, ep);
 875             return null;
 876         }
 877
 878         Action tundstAction;
 879
 880         if (tunDst.getIpv4Address() != null) {
 881             nextHop = tunDst.getIpv4Address().getValue();
 882             tundstAction = nxLoadTunIPv4Action(nextHop, false);
 883         } else if (tunDst.getIpv6Address() != null) {
 884             // nextHop = tunDst.getIpv6Address().getValue();
 885             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
 886                     ofc.getNodeId());
 887             return null;
 888         } else {
 889             // this shouldn't happen
 890             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
 891             return null;
 892         }
 893
 894         long portNum;
 895         try {
 896             portNum = getOfPortNum(tunPort);
 897         } catch (NumberFormatException ex) {
 898             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 899             return null;
 900         }
 901
 902         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 903         applyActions.add(tundstAction);
 904         // END TUNNEL
 905
 906         order += 1;
 907         applyActions.add(setdEPG);
 908         applyActions.add(setdCG);
 909         applyActions.add(setNextHop);
 910         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 911             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 912             .build();
 913         instructions.add(applyActionsIns);
 914
 915         applyActionsIns = new InstructionBuilder().setOrder(order++)
 916             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 917             .build();
 918
 919         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 920             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 921             .build();
 922         instructions.add(gotoTable);
 923
 924         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 925         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId())));
 926         Match match = mb.build();
 927         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL2", match);
 928         FlowBuilder flowb = base().setId(flowid)
 929             .setPriority(Integer.valueOf(50))
 930             .setMatch(match)
 931             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 932
 933         return flowb.build();
 934     }
 935
 936     private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId,
 937             EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc,
 938             Subnet srcSubnet) {
 939
 940         // TODO Li alagalah - refactor common code but keep simple method
 941
 942         // this endpoint is on a different switch; send to the
 943         // appropriate tunnel
 944         Subnet destSubnet = null;
 945         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 946         if (subnets == null) {
 947             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 948             return null;
 949         }
 950         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 951         for (Subnet subnet : subnets) {
 952             // TODO Li alagalah add IPv6 support
 953             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 954                 destSubnet = subnet;
 955                 break;
 956             }
 957         }
 958         if (destSubnet == null) {
 959             LOG.info("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 960             return null;
 961         }
 962
 963         if (destSubnet.getVirtualRouterIp() == null) {
 964             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 965                     destL3Address.getKey());
 966             return null;
 967         }
 968
 969         if (srcSubnet.getVirtualRouterIp() == null) {
 970             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 971             return null;
 972         }
 973         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 974         if (destL3c == null || destL3c.getId() == null) {
 975             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 976             return null;
 977         }
 978         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 979         if (srcL3c == null || srcL3c.getId() == null) {
 980             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 981             return null;
 982         }
 983
 984         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 985             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 986                 .getValue(), destL3c.getId().getValue());
 987             return null;
 988         }
 989
 990         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 991         MacAddress epDestMac = destEp.getMacAddress();
 992         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 993         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 994             // This is our final destination, so match on actual EP mac.
 995             matcherMac = epDestMac;
 996         }
 997         ArrayList<Instruction> l3instructions = new ArrayList<>();
 998         List<Action> applyActions = new ArrayList<>();
 999         List<Action> l3ApplyActions = new ArrayList<>();
1000
1001         int order = 0;
1002
1003         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
1004         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
1005         Action setNextHop;
1006         String nextHop;
1007
1008         // BEGIN TUNNEL HANDLING
1009         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
1010         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
1011         if (tunDst == null) {
1012             LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address);
1013             return null;
1014         }
1015         if (tunPort == null) {
1016             LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, destL3Address);
1017             return null;
1018         }
1019
1020         Action tundstAction;
1021
1022         if (tunDst.getIpv4Address() != null) {
1023             nextHop = tunDst.getIpv4Address().getValue();
1024             tundstAction = nxLoadTunIPv4Action(nextHop, false);
1025         } else if (tunDst.getIpv6Address() != null) {
1026             // nextHop = tunDst.getIpv6Address().getValue();
1027             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
1028                     ofc.getNodeId());
1029             return null;
1030         } else {
1031             // this shouldn't happen
1032             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
1033             return null;
1034         }
1035
1036         long portNum;
1037         try {
1038             portNum = getOfPortNum(tunPort);
1039         } catch (NumberFormatException ex) {
1040             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
1041             return null;
1042         }
1043
1044         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1045         applyActions.add(tundstAction);
1046         // END TUNNEL
1047
1048         order += 1;
1049         applyActions.add(setdEPG);
1050         applyActions.add(setdCG);
1051         applyActions.add(setNextHop);
1052
1053         // Lets not re-write the srcMac if its local.
1054         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
1055             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
1056             l3ApplyActions.add(setDlSrc);
1057         }
1058
1059         Action setDlDst = setDlDstAction(epDestMac);
1060         l3ApplyActions.add(setDlDst);
1061
1062         Action decTtl = decNwTtlAction();
1063         l3ApplyActions.add(decTtl);
1064
1065         applyActions.addAll(l3ApplyActions);
1066         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1067             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1068             .build();
1069
1070         l3instructions.add(applyActionsIns);
1071         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1072             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1073             .build();
1074         l3instructions.add(gotoTable);
1075         Layer3Match m = null;
1076         Long etherType = null;
1077         String ikey = null;
1078         if (destL3Address.getIpAddress().getIpv4Address() != null) {
1079             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
1080             etherType = IPv4;
1081             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
1082         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
1083             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
1084             etherType = IPv6;
1085             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
1086         } else {
1087             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
1088             return null;
1089         }
1090
1091         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
1092             .setLayer3Match(m);
1093         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id())));
1094         Match match = mb.build();
1095         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL3", match);
1096         FlowBuilder flowb = base().setId(flowid)
1097             .setPriority(Integer.valueOf(132))
1098             .setMatch(match)
1099             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
1100         return flowb.build();
1101     }
1102
1103     private NetworkDomainId getEPNetworkContainment(Endpoint endpoint) {
1104         if (endpoint.getNetworkContainment() != null) {
1105             return endpoint.getNetworkContainment();
1106         } else {
1107             /*
1108              * TODO: Be alagalah: Endpoint Refactor: This should be set on input
1109              * which we can't do because of the backwards way endpoints were
1110              * "architected".
1111              */
1112             return ctx.getTenant(endpoint.getTenant())
1113                 .getEndpointGroup(endpoint.getEndpointGroup())
1114                 .getNetworkDomain();
1115         }
1116     }
1117
1118     private HashSet<Subnet> getSubnets(final TenantId tenantId) {
1119
1120         if (ctx.getDataBroker() == null) {
1121             return null;
1122         }
1123
1124         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
1125         InstanceIdentifier<Tenant> tiid = TenantUtils.tenantIid(tenantId);
1126         Optional<Tenant> tenantInfo;
1127         try {
1128             tenantInfo = t.read(LogicalDatastoreType.CONFIGURATION, tiid).get();
1129         } catch (Exception e) {
1130             LOG.error("Could not read Tenant {}", tenantId, e);
1131             return null;
1132         } finally {
1133             t.close();
1134         }
1135
1136         if (!tenantInfo.isPresent()) {
1137             LOG.warn("Tenant {} not found", tenantId);
1138             return null;
1139         }
1140
1141         ForwardingContext fwCtx = tenantInfo.get().getForwardingContext();
1142         if (fwCtx == null || fwCtx.getSubnet() == null) {
1143             return new HashSet<>();
1144         }
1145         return new HashSet<>(fwCtx.getSubnet());
1146     }
1147
1148     // Need a method to get subnets for EPs attached to the node locally
1149     // to set the source Mac address for the router interface.
1150     private List<Subnet> getLocalSubnets(NodeId nodeId) {
1151         Collection<Endpoint> endpointsForNode = ctx.getEndpointManager().getEndpointsForNode(nodeId);
1152
1153         List<Subnet> localSubnets = new ArrayList<Subnet>();
1154
1155         for (Endpoint endpoint : endpointsForNode) {
1156             HashSet<Subnet> subnets = getSubnets(endpoint.getTenant());
1157             if (subnets == null) {
1158                 LOG.debug("No local subnets in tenant {} for EP {}.", endpoint.getTenant(), endpoint.getKey());
1159                 continue;
1160             }
1161             NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint);
1162             for (Subnet subnet : subnets) {
1163                 if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) {
1164                     localSubnets.add(subnet);
1165                 }
1166             }
1167         }
1168         return localSubnets;
1169     }
1170
1171     static byte[] bytesFromHexString(String values) {
1172         String target = "";
1173         if (values != null) {
1174             target = values;
1175         }
1176         String[] octets = target.split(":");
1177
1178         byte[] ret = new byte[octets.length];
1179         for (int i = 0; i < octets.length; i++) {
1180             ret[i] = Integer.valueOf(octets[i], 16).byteValue();
1181         }
1182         return ret;
1183     }
1184 }