renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java

   1 /*
   2  * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow;
  10
  11 import static com.google.common.base.Preconditions.checkNotNull;
  12 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP;
  13 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4;
  14 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6;
  15 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch;
  16 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns;
  17 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction;
  18 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch;
  19 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum;
  20 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.gotoTableIns;
  21 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.groupAction;
  22 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.instructions;
  23 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpOpAction;
  24 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpShaAction;
  25 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction;
  26 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction;
  27 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action;
  28 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction;
  29 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction;
  30 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction;
  31 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction;
  32 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction;
  33 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction;
  34 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction;
  35 import static org.opendaylight.groupbasedpolicy.util.DataStoreHelper.readFromDs;
  36
  37 import java.math.BigInteger;
  38 import java.util.ArrayList;
  39 import java.util.Collection;
  40 import java.util.Collections;
  41 import java.util.HashMap;
  42 import java.util.HashSet;
  43 import java.util.List;
  44 import java.util.Map;
  45 import java.util.Map.Entry;
  46 import java.util.Objects;
  47 import java.util.Set;
  48
  49 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  50 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  51 import org.opendaylight.groupbasedpolicy.dto.EgKey;
  52 import org.opendaylight.groupbasedpolicy.dto.EpKey;
  53 import org.opendaylight.groupbasedpolicy.dto.IndexedTenant;
  54 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext;
  55 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfWriter;
  56 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.endpoint.EndpointManager;
  57 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch;
  58 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals;
  59 import org.opendaylight.groupbasedpolicy.util.IidFactory;
  60 import org.opendaylight.groupbasedpolicy.util.TenantUtils;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
  63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.l3.prefix.fields.EndpointL3Gateways;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Prefix;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.ForwardingContext;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.L3Context;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.forwarding.context.Subnet;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.ethernet.match.fields.EthernetDestinationBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatchBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.Layer3Match;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 104 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 105 import org.slf4j.Logger;
 106 import org.slf4j.LoggerFactory;
 107
 108 import com.google.common.base.Optional;
 109 import com.google.common.base.Strings;
 110 import com.google.common.collect.HashMultimap;
 111 import com.google.common.collect.SetMultimap;
 112 import com.google.common.collect.Sets;
 113
 114 /**
 115  * Manage the table that maps the destination address to the next hop for the
 116  * path as well as applies any relevant routing transformations.
 117  */
 118 public class DestinationMapper extends FlowTable {
 119
 120     protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class);
 121
 122     // TODO Li alagalah: Improve UT coverage for this class.
 123
 124     // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows
 125     // This ensures we have the appropriate network-containment'
 126
 127     public static short TABLE_ID;
 128     /**
 129      * This is the MAC address of the magical router in the sky
 130      */
 131     public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5");
 132     public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00");
 133     public static final Integer BASE_L3_PRIORITY = 100;
 134
 135     public DestinationMapper(OfContext ctx, short tableId) {
 136         super(ctx);
 137         this.TABLE_ID = tableId;
 138     }
 139
 140     Map<TenantId, HashSet<Subnet>> subnetsByTenant = new HashMap<TenantId, HashSet<Subnet>>();
 141
 142     @Override
 143     public short getTableId() {
 144         return TABLE_ID;
 145     }
 146
 147     @Override
 148     public void sync(NodeId nodeId, OfWriter ofWriter) throws Exception {
 149
 150         TenantId currentTenant = null;
 151
 152         ofWriter.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null, TABLE_ID));
 153
 154         SetMultimap<EpKey, EpKey> visitedEps = HashMultimap.create();
 155         Set<EndpointFwdCtxOrdinals> epOrdSet = new HashSet<>();
 156
 157         for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) {
 158             Set<EndpointGroupId> srcEpgIds = new HashSet<>();
 159             if (srcEp.getEndpointGroup() != null)
 160                 srcEpgIds.add(srcEp.getEndpointGroup());
 161             if (srcEp.getEndpointGroups() != null)
 162                 srcEpgIds.addAll(srcEp.getEndpointGroups());
 163
 164             for (EndpointGroupId epgId : srcEpgIds) {
 165                 EgKey epg = new EgKey(srcEp.getTenant(), epgId);
 166                 Set<EgKey> peers = Sets.union(Collections.singleton(epg), ctx.getCurrentPolicy().getPeers(epg));
 167                 for (EgKey peer : peers) {
 168                     for (Endpoint peerEp : ctx.getEndpointManager().getEndpointsForGroup(peer)) {
 169                         currentTenant = peerEp.getTenant();
 170                         subnetsByTenant.put(currentTenant, getSubnets(currentTenant));
 171                         EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress());
 172                         EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress());
 173
 174                         if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) {
 175                             continue;
 176                         }
 177                         syncEP(ofWriter, nodeId, srcEp, peerEp);
 178                         visitedEps.put(srcEpKey, peerEpKey);
 179
 180                         // Process subnets and flood-domains for epPeer
 181                         EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx,
 182                                 peerEp);
 183                         if (epOrds == null) {
 184                             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", peerEp);
 185                             continue;
 186                         }
 187
 188                         epOrdSet.add(epOrds);
 189                     }
 190                 }
 191             }
 192         }
 193
 194         for (Entry<TenantId, HashSet<Subnet>> subnetEntry : subnetsByTenant.entrySet()) {
 195             if (subnetEntry.getValue() == null) {
 196                 LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey());
 197                 continue;
 198             }
 199             currentTenant = subnetEntry.getKey();
 200             for (Subnet sn : subnetEntry.getValue()) {
 201                 L3Context l3c = getL3ContextForSubnet(currentTenant, sn);
 202                 Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn,
 203                         OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId()));
 204                 if (arpFlow != null) {
 205                     ofWriter.writeFlow(nodeId, TABLE_ID, arpFlow);
 206                 } else {
 207                     LOG.debug(
 208                             "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .",
 209                             sn.getIpPrefix().getValue());
 210                 }
 211             }
 212         }
 213
 214         // Write broadcast flows per flood domain.
 215         for (EndpointFwdCtxOrdinals epOrd : epOrdSet) {
 216             if (ofWriter.groupExists(nodeId, Integer.valueOf(epOrd.getFdId()).longValue())) {
 217                 ofWriter.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd));
 218             }
 219         }
 220
 221         // L3 Prefix Endpoint handling
 222         Collection<EndpointL3Prefix> prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(currentTenant);
 223         if (prefixEps != null) {
 224             LOG.trace("DestinationMapper - Processing L3PrefixEndpoints");
 225             for (EndpointL3Prefix prefixEp : prefixEps) {
 226                 List<Subnet> localSubnets = getLocalSubnets(nodeId);
 227                 if (localSubnets == null) {
 228                     continue;
 229                 }
 230                 for (Subnet localSubnet: localSubnets) {
 231                     Flow prefixFlow = createL3PrefixFlow(prefixEp, nodeId, localSubnet);
 232                     if (prefixFlow != null) {
 233                         ofWriter.writeFlow(nodeId, TABLE_ID, prefixFlow);
 234                         LOG.trace("Wrote L3Prefix flow");
 235                     }
 236                 }
 237             }
 238         }
 239     }
 240
 241     // set up next-hop destinations for all the endpoints in the endpoint
 242     // group on the node
 243
 244     private Flow createL3PrefixFlow(EndpointL3Prefix prefixEp, NodeId nodeId, Subnet subnet) throws Exception {
 245         /*
 246          * Priority: 100+lengthprefix
 247          * Match: prefix, l3c, "mac address of router" ?
 248          * Action:
 249          * - set Reg2, Reg3 for L3Ep by L2Ep ?
 250          * - if external,
 251          * - Reg7: use switch location external port else punt for now
 252          * - if internal
 253          * - Reg7: grab L2Ep from L3Ep and use its location info
 254          * - goto_table: POLENF (will check there for external on EP)
 255          */
 256
 257         ReadOnlyTransaction rTx = ctx.getDataBroker().newReadOnlyTransaction();
 258         // TODO Bug #3440 Target: Be - should support for more than first gateway.
 259         EndpointL3Gateways l3Gateway = prefixEp.getEndpointL3Gateways().get(0);
 260         Optional<EndpointL3> optL3Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 261                 IidFactory.l3EndpointIid(l3Gateway.getL3Context(), l3Gateway.getIpAddress()), rTx);
 262         if (!optL3Ep.isPresent()) {
 263             LOG.error("createL3PrefixFlow - L3Endpoint gateway {} for L3Prefix {} not found.", l3Gateway, prefixEp);
 264             return null;
 265         }
 266         EndpointL3 l3Ep = optL3Ep.get();
 267         if (l3Ep.getL2Context() == null || l3Ep.getMacAddress() == null) {
 268             LOG.debug("L3 endpoint representing L3 gateway does not contain L2-context or MAC address. {}", l3Ep);
 269             return null;
 270         }
 271         Optional<Endpoint> optL2Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 272                 IidFactory.endpointIid(l3Ep.getL2Context(), l3Ep.getMacAddress()), rTx);
 273         if (!optL2Ep.isPresent()) {
 274             LOG.error("createL3PrefixFlow - L2Endpoint for L3Gateway {} not found.", l3Ep);
 275             return null;
 276         }
 277         Endpoint l2Ep = optL2Ep.get();
 278         EndpointFwdCtxOrdinals epFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, l2Ep);
 279         if (epFwdCtxOrds == null) {
 280             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", l2Ep);
 281             return null;
 282         }
 283
 284         NetworkDomainId epNetworkContainment = getEPNetworkContainment(l2Ep);
 285
 286         MacAddress epDestMac = l2Ep.getMacAddress();
 287         MacAddress destSubnetGatewayMac = l2Ep.getMacAddress();
 288         L3Context destL3c = getL3ContextForSubnet(prefixEp.getTenant(), subnet);
 289         if (destL3c == null || destL3c.getId() == null) {
 290             LOG.error("No L3 Context found associated with subnet {}", subnet.getId());
 291             return null;
 292         }
 293
 294         MacAddress matcherMac = routerPortMac(destL3c, subnet.getVirtualRouterIp());
 295
 296         ArrayList<Instruction> l3instructions = new ArrayList<>();
 297         List<Action> applyActions = new ArrayList<>();
 298         List<Action> l3ApplyActions = new ArrayList<>();
 299
 300         int order = 0;
 301
 302         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 303         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 304         Action setNextHop;
 305         String nextHop=null;
 306
 307         OfOverlayContext ofc = l2Ep.getAugmentation(OfOverlayContext.class);
 308
 309         long portNum = -1;
 310         if (EndpointManager.isInternal(l2Ep, ctx.getTenant(l2Ep.getTenant()).getExternalImplicitGroups())) {
 311             checkNotNull(ofc.getNodeConnectorId());
 312             nextHop = ofc.getNodeConnectorId().getValue();
 313             try {
 314                 portNum = getOfPortNum(ofc.getNodeConnectorId());
 315             } catch (NumberFormatException ex) {
 316                 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 317                 return null;
 318             }
 319
 320         } else {
 321             // External
 322             Set<NodeConnectorId> externalPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
 323             checkNotNull(externalPorts);
 324             for (NodeConnectorId externalPort : externalPorts) {
 325                 // TODO Bug #3440 Target: Be - should support for more than first external port.
 326                 //TODO Bug 3546 - Difficult: External port is unrelated to Tenant, L3C, L2BD..
 327                 nextHop = externalPort.getValue();
 328                 try {
 329                     portNum = getOfPortNum(externalPort);
 330                 } catch (NumberFormatException ex) {
 331                     LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 332                     return null;
 333                 }
 334                 continue;
 335             }
 336         }
 337
 338         if (Strings.isNullOrEmpty(nextHop)
 339                 || portNum == -1) {
 340             LOG.error("createL3Prefix - Cannot find nodeConnectorId for {} for Prefix: ", l2Ep, prefixEp);
 341             return null;
 342         }
 343         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 344
 345         Action setDlDst = setDlDstAction(epDestMac);
 346         l3ApplyActions.add(setDlDst);
 347
 348         Action decTtl = decNwTtlAction();
 349         l3ApplyActions.add(decTtl);
 350
 351         order += 1;
 352         applyActions.add(setdEPG);
 353         applyActions.add(setdCG);
 354         applyActions.add(setNextHop);
 355
 356         applyActions.addAll(l3ApplyActions);
 357         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 358             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 359             .build();
 360
 361         l3instructions.add(applyActionsIns);
 362         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 363             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 364             .build();
 365         l3instructions.add(gotoTable);
 366
 367         Layer3Match m = null;
 368         Long etherType = null;
 369         String ikey = null;
 370         Integer prefixLength=0;
 371         if (prefixEp.getIpPrefix().getIpv4Prefix() != null) {
 372             ikey = prefixEp.getIpPrefix().getIpv4Prefix().getValue();
 373             etherType = IPv4;
 374             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv4Prefix().getValue().split("/")[1]);
 375             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 376         } else if (prefixEp.getIpPrefix().getIpv6Prefix() != null) {
 377             ikey = prefixEp.getIpPrefix().getIpv6Prefix().getValue();
 378             etherType = IPv6;
 379             /*
 380              *  This will result in flows with priority between 100-228, but since its matching on IPv6 prefix as well
 381              *  this shouldn't pose and issue, as the priority is more important within the address space of the matcher,
 382              *  even though technically flows are processed in priority order.
 383              */
 384
 385             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv6Prefix().getValue().split("/")[1]);
 386             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 387         } else {
 388             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", prefixEp);
 389             return null;
 390         }
 391
 392         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType));
 393         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 394         Match match = mb.build();
 395         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "L3prefix", match);
 396         FlowBuilder flowb = base().setId(flowid)
 397             .setPriority(Integer.valueOf(BASE_L3_PRIORITY+prefixLength))
 398             .setMatch(match)
 399             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 400         return flowb.build();
 401     }
 402
 403     private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) {
 404         MatchBuilder mb = new MatchBuilder()
 405                 .setEthernetMatch(new EthernetMatchBuilder().setEthernetDestination(
 406                         new EthernetDestinationBuilder().setAddress(MULTICAST_MAC)
 407                                 .setMask(MULTICAST_MAC)
 408                                 .build()).build());
 409         addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId())));
 410
 411         Match match = mb.build();
 412         FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "broadcast", match);
 413         FlowBuilder flowb = base().setPriority(Integer.valueOf(140))
 414             .setId(flowId)
 415             .setMatch(match)
 416             .setInstructions(
 417                     instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false),
 418                             groupAction(Long.valueOf(epOrd.getFdId())))));
 419
 420         return flowb.build();
 421     }
 422
 423     private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) {
 424
 425         if (ctx.getDataBroker() == null) {
 426             return null;
 427         }
 428
 429         MacAddress defaultMacAddress = ROUTER_MAC;
 430
 431         EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId());
 432         InstanceIdentifier<EndpointL3> endpointsIid = InstanceIdentifier.builder(Endpoints.class)
 433             .child(EndpointL3.class, l3Key)
 434             .build();
 435         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 436
 437         Optional<EndpointL3> r;
 438         try {
 439             r = t.read(LogicalDatastoreType.OPERATIONAL, endpointsIid).get();
 440             if (!r.isPresent())
 441                 return defaultMacAddress;
 442             EndpointL3 epL3 = r.get();
 443             if (epL3.getMacAddress() == null) {
 444                 return defaultMacAddress;
 445             } else {
 446                 return epL3.getMacAddress();
 447             }
 448         } catch (Exception e) {
 449             LOG.error("Error reading EndpointL3 {}.{}", l3c, ipAddress, e);
 450             return null;
 451         }
 452     }
 453
 454     private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) {
 455         IndexedTenant indexedTenant = ctx.getTenant(tenantId);
 456         if (indexedTenant == null) {
 457             LOG.debug("Tenant {} is null, cannot get L3 context", tenantId);
 458             return null;
 459         }
 460         L3Context l3c = indexedTenant.resolveL3Context(sn.getId());
 461         return l3c;
 462     }
 463
 464     private Flow createRouterArpFlow(TenantId tenantId, NodeId nodeId, Subnet sn, int l3Id) {
 465         if (sn == null || sn.getVirtualRouterIp() == null) {
 466             LOG.trace("Didn't create routerArpFlow since either subnet or subnet virtual router was null");
 467             return null;
 468         }
 469         /*
 470          * TODO: Li alagalah: This should be new Yang "gateways" list as well,
 471          * that expresses the gateway and prefixes it is interface for. Should
 472          * also check for external.
 473          */
 474         if (sn.getVirtualRouterIp().getIpv4Address() != null) {
 475             String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue();
 476
 477             L3Context l3c = getL3ContextForSubnet(tenantId, sn);
 478             if (l3c == null) {
 479                 LOG.error("No L3 Context found associated with subnet {}", sn.getId());
 480             }
 481
 482             MacAddress routerMac = routerPortMac(l3c, sn.getVirtualRouterIp());
 483             if (routerMac == null) {
 484                 return null;
 485             }
 486
 487             BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue()));
 488
 489             MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match(
 490                     new ArpMatchBuilder().setArpOp(Integer.valueOf(1))
 491                         .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32"))
 492                         .build());
 493             addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id)));
 494
 495             Match match = mb.build();
 496             FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "routerarp", match);
 497             FlowBuilder flowb = base().setPriority(150)
 498                 .setId(flowId)
 499                 .setMatch(match)
 500                 .setInstructions(
 501                         instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac),
 502                                 nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(),
 503                                 nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(),
 504                                 nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue()
 505                                         + ":INPORT")))));
 506             return flowb.build();
 507         } else {
 508             LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId()
 509                 .getValue());
 510             return null;
 511         }
 512
 513     }
 514
 515     private Flow createLocalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) {
 516
 517         // TODO Li alagalah - refactor common code but keep simple method
 518         ArrayList<Instruction> instructions = new ArrayList<>();
 519         List<Action> applyActions = new ArrayList<>();
 520
 521         int order = 0;
 522
 523         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 524         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 525         Action setNextHop;
 526         String nextHop;
 527
 528         // BEGIN L2 LOCAL
 529         nextHop = ofc.getNodeConnectorId().getValue();
 530
 531         long portNum;
 532         try {
 533             portNum = getOfPortNum(ofc.getNodeConnectorId());
 534         } catch (NumberFormatException ex) {
 535             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 536             return null;
 537         }
 538
 539         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 540
 541         // END L2 LOCAL
 542
 543         order += 1;
 544         applyActions.add(setdEPG);
 545         applyActions.add(setdCG);
 546         applyActions.add(setNextHop);
 547         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 548             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 549             .build();
 550         instructions.add(applyActionsIns);
 551
 552         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 553             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 554             .build();
 555         instructions.add(gotoTable);
 556
 557         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 558         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
 559         Match match = mb.build();
 560         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL2", match);
 561         FlowBuilder flowb = base().setId(flowid)
 562             .setPriority(Integer.valueOf(50))
 563             .setMatch(match)
 564             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 565         return flowb.build();
 566     }
 567
 568     private void syncEP(OfWriter ofWriter, NodeId nodeId, Endpoint srcEp, Endpoint destEp)
 569             throws Exception {
 570
 571         if (ctx.getTenant(srcEp.getTenant()) == null
 572                 || ctx.getTenant(destEp.getTenant()) == null) {
 573             LOG.debug("Source or destination EP references empty tenant srcEp:{} destEp:{}", srcEp, destEp);
 574             return;
 575         }
 576
 577         // TODO: Conditions messed up, but for now, send policyInfo until this
 578         // is fixed.
 579         EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, destEp);
 580         if (destEpFwdCtxOrds == null) {
 581             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", destEp);
 582             return;
 583         }
 584         EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, srcEp);
 585         if (srcEpFwdCtxOrds == null) {
 586             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", srcEp);
 587             return;
 588         }
 589
 590
 591         if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) {
 592             if (destEp.getTenant() == null) {
 593                 LOG.debug("Didn't process endpoint {} due to tenant being null", destEp.getKey());
 594             } else {
 595                 LOG.debug("Didn't process endpoint {} due to EPG(s) being null", destEp.getKey());
 596             }
 597             return;
 598         }
 599
 600         if (EndpointManager.isExternal(destEp, ctx.getTenant(destEp.getTenant()).getExternalImplicitGroups())) {
 601             LOG.error("syncEp(): External endpoints should not be seen here.");
 602             return;
 603         }
 604
 605         /*
 606          * Only care about subnets for L3, but fetch them before loop. We need
 607          * the local subnets for setting SRC MAC for routing. All Routing is now
 608          * done locally! YAY! Instead of being shovelled L2 style across network
 609          * ala Helium.
 610          */
 611         List<Subnet> localSubnets = getLocalSubnets(nodeId);
 612         if (localSubnets == null) {
 613             LOG.error("No subnets could be found locally for node: {}", nodeId);
 614             return;
 615         }
 616
 617         OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class);
 618         if (Objects.equals(ofc.getNodeId(), nodeId)) {
 619             // this is a local endpoint; send to the approppriate local
 620             // port
 621
 622             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 623                 ofWriter.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc));
 624             }
 625             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 626             // The Endpoint conflation must end!
 627             if (destEp.getL3Address() == null) {
 628                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 629                 return;
 630             }
 631
 632             for (L3Address l3a : destEp.getL3Address()) {
 633                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 634                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 635                             destEp.getL3Address());
 636                     continue;
 637                 } else {
 638                     for (Subnet localSubnet : localSubnets) {
 639                         Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet);
 640                         if (flow != null) {
 641                             ofWriter.writeFlow(nodeId, TABLE_ID, flow);
 642                         } else {
 643                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 644                                     localSubnet.getIpPrefix().getValue());
 645                         }
 646                     }
 647                 }
 648             }
 649         } else {
 650             // this endpoint is on a different switch; send to the
 651             // appropriate tunnel
 652             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 653                 Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc);
 654                 if (remoteL2Flow != null) {
 655                     ofWriter.writeFlow(nodeId, TABLE_ID, remoteL2Flow);
 656                 }
 657             } else {
 658                 LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}",
 659                         srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId());
 660             }
 661
 662             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 663             // The Endpoint conflation must end!
 664             if (destEp.getL3Address() == null) {
 665                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 666                 return;
 667             }
 668             for (L3Address l3a : destEp.getL3Address()) {
 669                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 670                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 671                             destEp.getL3Address());
 672                     continue;
 673                 } else {
 674                     for (Subnet localSubnet : localSubnets) {
 675                         Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds,
 676                                 destEpFwdCtxOrds, ofc, localSubnet);
 677                         if (remoteL3Flow != null) {
 678                             ofWriter.writeFlow(nodeId, TABLE_ID, remoteL3Flow);
 679                         } else {
 680                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 681                                     localSubnet.getIpPrefix().getValue());
 682                         }
 683                     }
 684                 }
 685             }
 686         } // remote (tunnel)
 687
 688         // }
 689
 690     }
 691
 692     /*
 693      * ################################## DestMapper Flow methods
 694      * ##################################
 695      */
 696     private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
 697             OfOverlayContext ofc, Subnet srcSubnet) {
 698
 699         // TODO Li alagalah - refactor common code but keep simple method
 700
 701         Subnet destSubnet = null;
 702         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 703         if (subnets == null) {
 704             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 705             return null;
 706         }
 707         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 708         for (Subnet subnet : subnets) {
 709             // TODO Li alagalah add IPv6 support
 710             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 711                 destSubnet = subnet;
 712                 break;
 713             }
 714         }
 715         if (destSubnet == null) {
 716             LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 717             return null;
 718         }
 719
 720         if (destSubnet.getVirtualRouterIp() == null) {
 721             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 722                     destL3Address.getKey());
 723             return null;
 724         }
 725
 726         if (srcSubnet.getVirtualRouterIp() == null) {
 727             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 728             return null;
 729         }
 730         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 731         if (destL3c == null || destL3c.getId() == null) {
 732             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 733             return null;
 734         }
 735         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 736         if (srcL3c == null || srcL3c.getId() == null) {
 737             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 738             return null;
 739         }
 740
 741         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 742             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 743                 .getValue(), destL3c.getId().getValue());
 744             return null;
 745         }
 746
 747         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 748         MacAddress epDestMac = destEp.getMacAddress();
 749         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 750
 751         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 752             // This is our final destination, so match on actual EP mac.
 753             matcherMac = epDestMac;
 754         }
 755
 756         ArrayList<Instruction> l3instructions = new ArrayList<>();
 757         List<Action> applyActions = new ArrayList<>();
 758         List<Action> l3ApplyActions = new ArrayList<>();
 759
 760         int order = 0;
 761
 762         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 763         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 764         Action setNextHop;
 765         String nextHop;
 766
 767         // BEGIN L3 LOCAL
 768         nextHop = ofc.getNodeConnectorId().getValue();
 769
 770         long portNum;
 771         try {
 772             portNum = getOfPortNum(ofc.getNodeConnectorId());
 773         } catch (NumberFormatException ex) {
 774             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 775             return null;
 776         }
 777
 778         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 779         // END L3 LOCAL
 780
 781         // Lets not re-write the srcMac if its local.
 782         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
 783             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
 784             l3ApplyActions.add(setDlSrc);
 785         }
 786
 787         Action setDlDst = setDlDstAction(epDestMac);
 788         l3ApplyActions.add(setDlDst);
 789
 790         Action decTtl = decNwTtlAction();
 791         l3ApplyActions.add(decTtl);
 792
 793         order += 1;
 794         applyActions.add(setdEPG);
 795         applyActions.add(setdCG);
 796         applyActions.add(setNextHop);
 797
 798         applyActions.addAll(l3ApplyActions);
 799         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 800             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 801             .build();
 802
 803         l3instructions.add(applyActionsIns);
 804         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 805             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 806             .build();
 807         l3instructions.add(gotoTable);
 808         Layer3Match m = null;
 809         Long etherType = null;
 810         String ikey = null;
 811         if (destL3Address.getIpAddress().getIpv4Address() != null) {
 812             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
 813             etherType = IPv4;
 814             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 815         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
 816             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
 817             etherType = IPv6;
 818             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 819         } else {
 820             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
 821             return null;
 822         }
 823
 824         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
 825             .setLayer3Match(m);
 826         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 827         Match match = mb.build();
 828         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL3", match);
 829         FlowBuilder flowb = base().setId(flowid)
 830             .setPriority(Integer.valueOf(132))
 831             .setMatch(match)
 832             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 833         return flowb.build();
 834     }
 835
 836     private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds,
 837             EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) {
 838
 839         // TODO Li alagalah - refactor common code but keep simple method
 840
 841         // this endpoint is on a different switch; send to the
 842         // appropriate tunnel
 843
 844         ArrayList<Instruction> instructions = new ArrayList<>();
 845         List<Action> applyActions = new ArrayList<>();
 846
 847         int order = 0;
 848
 849         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
 850         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
 851         Action setNextHop;
 852         String nextHop;
 853
 854         // BEGIN TUNNEL HANDLING
 855         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
 856         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
 857         if (tunDst == null) {
 858             LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep);
 859             return null;
 860         }
 861         if (tunPort == null) {
 862             LOG.warn("Failed to get Tunnel Port for NodeId {} with EP {}", nodeId, ep);
 863             return null;
 864         }
 865
 866         Action tundstAction;
 867
 868         if (tunDst.getIpv4Address() != null) {
 869             nextHop = tunDst.getIpv4Address().getValue();
 870             tundstAction = nxLoadTunIPv4Action(nextHop, false);
 871         } else if (tunDst.getIpv6Address() != null) {
 872             // nextHop = tunDst.getIpv6Address().getValue();
 873             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
 874                     ofc.getNodeId());
 875             return null;
 876         } else {
 877             // this shouldn't happen
 878             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
 879             return null;
 880         }
 881
 882         long portNum;
 883         try {
 884             portNum = getOfPortNum(tunPort);
 885         } catch (NumberFormatException ex) {
 886             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 887             return null;
 888         }
 889
 890         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 891         applyActions.add(tundstAction);
 892         // END TUNNEL
 893
 894         order += 1;
 895         applyActions.add(setdEPG);
 896         applyActions.add(setdCG);
 897         applyActions.add(setNextHop);
 898         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 899             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 900             .build();
 901         instructions.add(applyActionsIns);
 902
 903         applyActionsIns = new InstructionBuilder().setOrder(order++)
 904             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 905             .build();
 906
 907         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 908             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 909             .build();
 910         instructions.add(gotoTable);
 911
 912         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 913         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId())));
 914         Match match = mb.build();
 915         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL2", match);
 916         FlowBuilder flowb = base().setId(flowid)
 917             .setPriority(Integer.valueOf(50))
 918             .setMatch(match)
 919             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 920
 921         return flowb.build();
 922     }
 923
 924     private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId,
 925             EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc,
 926             Subnet srcSubnet) {
 927
 928         // TODO Li alagalah - refactor common code but keep simple method
 929
 930         // this endpoint is on a different switch; send to the
 931         // appropriate tunnel
 932         Subnet destSubnet = null;
 933         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 934         if (subnets == null) {
 935             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 936             return null;
 937         }
 938         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 939         for (Subnet subnet : subnets) {
 940             // TODO Li alagalah add IPv6 support
 941             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 942                 destSubnet = subnet;
 943                 break;
 944             }
 945         }
 946         if (destSubnet == null) {
 947             LOG.info("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 948             return null;
 949         }
 950
 951         if (destSubnet.getVirtualRouterIp() == null) {
 952             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 953                     destL3Address.getKey());
 954             return null;
 955         }
 956
 957         if (srcSubnet.getVirtualRouterIp() == null) {
 958             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 959             return null;
 960         }
 961         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 962         if (destL3c == null || destL3c.getId() == null) {
 963             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 964             return null;
 965         }
 966         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 967         if (srcL3c == null || srcL3c.getId() == null) {
 968             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 969             return null;
 970         }
 971
 972         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 973             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 974                 .getValue(), destL3c.getId().getValue());
 975             return null;
 976         }
 977
 978         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 979         MacAddress epDestMac = destEp.getMacAddress();
 980         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 981         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 982             // This is our final destination, so match on actual EP mac.
 983             matcherMac = epDestMac;
 984         }
 985         ArrayList<Instruction> l3instructions = new ArrayList<>();
 986         List<Action> applyActions = new ArrayList<>();
 987         List<Action> l3ApplyActions = new ArrayList<>();
 988
 989         int order = 0;
 990
 991         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
 992         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
 993         Action setNextHop;
 994         String nextHop;
 995
 996         // BEGIN TUNNEL HANDLING
 997         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
 998         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
 999         if (tunDst == null) {
1000             LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address);
1001             return null;
1002         }
1003         if (tunPort == null) {
1004             LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, destL3Address);
1005             return null;
1006         }
1007
1008         Action tundstAction;
1009
1010         if (tunDst.getIpv4Address() != null) {
1011             nextHop = tunDst.getIpv4Address().getValue();
1012             tundstAction = nxLoadTunIPv4Action(nextHop, false);
1013         } else if (tunDst.getIpv6Address() != null) {
1014             // nextHop = tunDst.getIpv6Address().getValue();
1015             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
1016                     ofc.getNodeId());
1017             return null;
1018         } else {
1019             // this shouldn't happen
1020             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
1021             return null;
1022         }
1023
1024         long portNum;
1025         try {
1026             portNum = getOfPortNum(tunPort);
1027         } catch (NumberFormatException ex) {
1028             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
1029             return null;
1030         }
1031
1032         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1033         applyActions.add(tundstAction);
1034         // END TUNNEL
1035
1036         order += 1;
1037         applyActions.add(setdEPG);
1038         applyActions.add(setdCG);
1039         applyActions.add(setNextHop);
1040
1041         // Lets not re-write the srcMac if its local.
1042         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
1043             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
1044             l3ApplyActions.add(setDlSrc);
1045         }
1046
1047         Action setDlDst = setDlDstAction(epDestMac);
1048         l3ApplyActions.add(setDlDst);
1049
1050         Action decTtl = decNwTtlAction();
1051         l3ApplyActions.add(decTtl);
1052
1053         applyActions.addAll(l3ApplyActions);
1054         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1055             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1056             .build();
1057
1058         l3instructions.add(applyActionsIns);
1059         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1060             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1061             .build();
1062         l3instructions.add(gotoTable);
1063         Layer3Match m = null;
1064         Long etherType = null;
1065         String ikey = null;
1066         if (destL3Address.getIpAddress().getIpv4Address() != null) {
1067             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
1068             etherType = IPv4;
1069             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
1070         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
1071             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
1072             etherType = IPv6;
1073             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
1074         } else {
1075             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
1076             return null;
1077         }
1078
1079         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
1080             .setLayer3Match(m);
1081         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id())));
1082         Match match = mb.build();
1083         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL3", match);
1084         FlowBuilder flowb = base().setId(flowid)
1085             .setPriority(Integer.valueOf(132))
1086             .setMatch(match)
1087             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
1088         return flowb.build();
1089     }
1090
1091     private NetworkDomainId getEPNetworkContainment(Endpoint endpoint) {
1092         if (endpoint.getNetworkContainment() != null) {
1093             return endpoint.getNetworkContainment();
1094         } else {
1095             /*
1096              * TODO: Be alagalah: Endpoint Refactor: This should be set on input
1097              * which we can't do because of the backwards way endpoints were
1098              * "architected".
1099              */
1100             return ctx.getTenant(endpoint.getTenant())
1101                 .getEndpointGroup(endpoint.getEndpointGroup())
1102                 .getNetworkDomain();
1103         }
1104     }
1105
1106     private HashSet<Subnet> getSubnets(final TenantId tenantId) {
1107
1108         if (ctx.getDataBroker() == null) {
1109             return null;
1110         }
1111
1112         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
1113         InstanceIdentifier<Tenant> tiid = TenantUtils.tenantIid(tenantId);
1114         Optional<Tenant> tenantInfo;
1115         try {
1116             tenantInfo = t.read(LogicalDatastoreType.CONFIGURATION, tiid).get();
1117         } catch (Exception e) {
1118             LOG.error("Could not read Tenant {}", tenantId, e);
1119             return null;
1120         } finally {
1121             t.close();
1122         }
1123
1124         if (!tenantInfo.isPresent()) {
1125             LOG.warn("Tenant {} not found", tenantId);
1126             return null;
1127         }
1128
1129         ForwardingContext fwCtx = tenantInfo.get().getForwardingContext();
1130         if (fwCtx == null || fwCtx.getSubnet() == null) {
1131             return new HashSet<>();
1132         }
1133         return new HashSet<>(fwCtx.getSubnet());
1134     }
1135
1136     // Need a method to get subnets for EPs attached to the node locally
1137     // to set the source Mac address for the router interface.
1138     private List<Subnet> getLocalSubnets(NodeId nodeId) {
1139         Collection<Endpoint> endpointsForNode = ctx.getEndpointManager().getEndpointsForNode(nodeId);
1140
1141         List<Subnet> localSubnets = new ArrayList<Subnet>();
1142
1143         for (Endpoint endpoint : endpointsForNode) {
1144             HashSet<Subnet> subnets = getSubnets(endpoint.getTenant());
1145             if (subnets == null) {
1146                 LOG.debug("No local subnets in tenant {} for EP {}.", endpoint.getTenant(), endpoint.getKey());
1147                 continue;
1148             }
1149             NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint);
1150             for (Subnet subnet : subnets) {
1151                 if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) {
1152                     localSubnets.add(subnet);
1153                 }
1154             }
1155         }
1156         return localSubnets;
1157     }
1158
1159     static byte[] bytesFromHexString(String values) {
1160         String target = "";
1161         if (values != null) {
1162             target = values;
1163         }
1164         String[] octets = target.split(":");
1165
1166         byte[] ret = new byte[octets.length];
1167         for (int i = 0; i < octets.length; i++) {
1168             ret[i] = Integer.valueOf(octets[i], 16).byteValue();
1169         }
1170         return ret;
1171     }
1172 }