renderers/ofoverlay/src/main/java/org/opendaylight/groupbasedpolicy/renderer/ofoverlay/flow/DestinationMapper.java

   1 /*
   2  * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow;
  10
  11 import com.google.common.base.Optional;
  12 import com.google.common.base.Strings;
  13 import com.google.common.collect.HashMultimap;
  14 import com.google.common.collect.SetMultimap;
  15 import com.google.common.collect.Sets;
  16 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  17 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  18 import org.opendaylight.groupbasedpolicy.endpoint.EpKey;
  19 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfWriter;
  20 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.OfContext;
  21 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.RegMatch;
  22 import org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.OrdinalFactory.EndpointFwdCtxOrdinals;
  23 import org.opendaylight.groupbasedpolicy.resolver.EgKey;
  24 import org.opendaylight.groupbasedpolicy.resolver.IndexedTenant;
  25 import org.opendaylight.groupbasedpolicy.resolver.PolicyInfo;
  26 import org.opendaylight.groupbasedpolicy.resolver.TenantUtils;
  27 import org.opendaylight.groupbasedpolicy.util.IidFactory;
  28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  29 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
  30 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv6Prefix;
  31 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.Match;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.EndpointGroupId;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.Endpoints;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.fields.L3Address;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoint.l3.prefix.fields.EndpointL3Gateways;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.Endpoint;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Key;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.endpoint.rev140421.endpoints.EndpointL3Prefix;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.EndpointLocation.LocationType;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.ofoverlay.rev140528.OfOverlayContext;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.Tenant;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.L3Context;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.policy.rev140421.tenants.tenant.Subnet;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.ethernet.match.fields.EthernetDestinationBuilder;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.EthernetMatchBuilder;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.Layer3Match;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.ArpMatchBuilder;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv4MatchBuilder;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.model.match.types.rev131026.match.layer._3.match.Ipv6MatchBuilder;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg2;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg3;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg4;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg5;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg7;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
  71 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  72 import org.slf4j.Logger;
  73 import org.slf4j.LoggerFactory;
  74
  75 import java.math.BigInteger;
  76 import java.util.ArrayList;
  77 import java.util.Collection;
  78 import java.util.Collections;
  79 import java.util.HashMap;
  80 import java.util.HashSet;
  81 import java.util.List;
  82 import java.util.Map;
  83 import java.util.Map.Entry;
  84 import java.util.Objects;
  85 import java.util.Set;
  86
  87 import static com.google.common.base.Preconditions.checkNotNull;
  88 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ARP;
  89 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv4;
  90 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.IPv6;
  91 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.addNxRegMatch;
  92 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.applyActionIns;
  93 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.decNwTtlAction;
  94 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.ethernetMatch;
  95 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.getOfPortNum;
  96 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.gotoTableIns;
  97 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.groupAction;
  98 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.instructions;
  99 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpOpAction;
 100 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpShaAction;
 101 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadArpSpaAction;
 102 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadRegAction;
 103 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIPv4Action;
 104 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxLoadTunIdAction;
 105 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpShaToArpThaAction;
 106 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveArpSpaToArpTpaAction;
 107 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.nxMoveEthSrcToEthDstAction;
 108 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.outputAction;
 109 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlDstAction;
 110 import static org.opendaylight.groupbasedpolicy.renderer.ofoverlay.flow.FlowUtils.setDlSrcAction;
 111 import static org.opendaylight.groupbasedpolicy.util.DataStoreHelper.readFromDs;
 112
 113 /**
 114  * Manage the table that maps the destination address to the next hop for the
 115  * path as well as applies any relevant routing transformations.
 116  */
 117 public class DestinationMapper extends FlowTable {
 118
 119     protected static final Logger LOG = LoggerFactory.getLogger(DestinationMapper.class);
 120
 121     // TODO Li alagalah: Improve UT coverage for this class.
 122
 123     // TODO Li alagalah: Use EndpointL3 for L3 flows, Endpoint for L2 flows
 124     // This ensures we have the appropriate network-containment'
 125
 126     public static short TABLE_ID;
 127     /**
 128      * This is the MAC address of the magical router in the sky
 129      */
 130     public static final MacAddress ROUTER_MAC = new MacAddress("88:f0:31:b5:12:b5");
 131     public static final MacAddress MULTICAST_MAC = new MacAddress("01:00:00:00:00:00");
 132     public static final Integer BASE_L3_PRIORITY = 100;
 133
 134     public DestinationMapper(OfContext ctx, short tableId) {
 135         super(ctx);
 136         this.TABLE_ID = tableId;
 137     }
 138
 139     Map<TenantId, HashSet<Subnet>> subnetsByTenant = new HashMap<TenantId, HashSet<Subnet>>();
 140
 141     @Override
 142     public short getTableId() {
 143         return TABLE_ID;
 144     }
 145
 146     @Override
 147     public void sync(NodeId nodeId, PolicyInfo policyInfo, OfWriter ofWriter) throws Exception {
 148
 149         TenantId currentTenant = null;
 150
 151         ofWriter.writeFlow(nodeId, TABLE_ID, dropFlow(Integer.valueOf(1), null, TABLE_ID));
 152
 153         SetMultimap<EpKey, EpKey> visitedEps = HashMultimap.create();
 154         Set<EndpointFwdCtxOrdinals> epOrdSet = new HashSet<>();
 155
 156         for (Endpoint srcEp : ctx.getEndpointManager().getEndpointsForNode(nodeId)) {
 157             Set<EndpointGroupId> srcEpgIds = new HashSet<>();
 158             if (srcEp.getEndpointGroup() != null)
 159                 srcEpgIds.add(srcEp.getEndpointGroup());
 160             if (srcEp.getEndpointGroups() != null)
 161                 srcEpgIds.addAll(srcEp.getEndpointGroups());
 162
 163             for (EndpointGroupId epgId : srcEpgIds) {
 164                 EgKey epg = new EgKey(srcEp.getTenant(), epgId);
 165                 Set<EgKey> peers = Sets.union(Collections.singleton(epg), policyInfo.getPeers(epg));
 166                 for (EgKey peer : peers) {
 167                     for (Endpoint peerEp : ctx.getEndpointManager().getEndpointsForGroup(peer)) {
 168                         currentTenant = peerEp.getTenant();
 169                         subnetsByTenant.put(currentTenant, getSubnets(currentTenant));
 170                         EpKey srcEpKey = new EpKey(srcEp.getL2Context(), srcEp.getMacAddress());
 171                         EpKey peerEpKey = new EpKey(peerEp.getL2Context(), peerEp.getMacAddress());
 172
 173                         if (visitedEps.get(srcEpKey) != null && visitedEps.get(srcEpKey).contains(peerEpKey)) {
 174                             continue;
 175                         }
 176                         syncEP(ofWriter, nodeId, policyInfo, srcEp, peerEp);
 177                         visitedEps.put(srcEpKey, peerEpKey);
 178
 179                         // Process subnets and flood-domains for epPeer
 180                         EndpointFwdCtxOrdinals epOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo,
 181                                 peerEp);
 182                         if (epOrds == null) {
 183                             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", peerEp);
 184                             continue;
 185                         }
 186
 187                         epOrdSet.add(epOrds);
 188                     }
 189                 }
 190             }
 191         }
 192
 193         for (Entry<TenantId, HashSet<Subnet>> subnetEntry : subnetsByTenant.entrySet()) {
 194             if (subnetEntry.getValue() == null) {
 195                 LOG.trace("Tenant: {} has empty subnet entry.", subnetEntry.getKey());
 196                 continue;
 197             }
 198             currentTenant = subnetEntry.getKey();
 199             for (Subnet sn : subnetEntry.getValue()) {
 200                 L3Context l3c = getL3ContextForSubnet(currentTenant, sn);
 201                 Flow arpFlow = createRouterArpFlow(currentTenant, nodeId, sn,
 202                         OrdinalFactory.getContextOrdinal(currentTenant, l3c.getId()));
 203                 if (arpFlow != null) {
 204                     ofWriter.writeFlow(nodeId, TABLE_ID, arpFlow);
 205                 } else {
 206                     LOG.debug(
 207                             "Gateway ARP flow is not created, because virtual router IP has not been set for subnet {} .",
 208                             sn.getIpPrefix().getValue());
 209                 }
 210             }
 211         }
 212
 213         // Write broadcast flows per flood domain.
 214         for (EndpointFwdCtxOrdinals epOrd : epOrdSet) {
 215             if (ofWriter.groupExists(nodeId, Integer.valueOf(epOrd.getFdId()).longValue())) {
 216                 ofWriter.writeFlow(nodeId, TABLE_ID, createBroadcastFlow(epOrd));
 217             }
 218         }
 219
 220         // L3 Prefix Endpoint handling
 221         Collection<EndpointL3Prefix> prefixEps = ctx.getEndpointManager().getEndpointsL3PrefixForTenant(currentTenant);
 222         if (prefixEps != null) {
 223             LOG.trace("DestinationMapper - Processing L3PrefixEndpoints");
 224             for (EndpointL3Prefix prefixEp : prefixEps) {
 225                 List<Subnet> localSubnets = getLocalSubnets(nodeId);
 226                 if (localSubnets == null) {
 227                     continue;
 228                 }
 229                 for (Subnet localSubnet: localSubnets) {
 230                     Flow prefixFlow = createL3PrefixFlow(prefixEp, policyInfo, nodeId, localSubnet);
 231                     if (prefixFlow != null) {
 232                         ofWriter.writeFlow(nodeId, TABLE_ID, prefixFlow);
 233                         LOG.trace("Wrote L3Prefix flow");
 234                     }
 235                 }
 236             }
 237         }
 238     }
 239
 240     // set up next-hop destinations for all the endpoints in the endpoint
 241     // group on the node
 242
 243     private Flow createL3PrefixFlow(EndpointL3Prefix prefixEp, PolicyInfo policyInfo, NodeId nodeId, Subnet subnet) throws Exception {
 244         /*
 245          * Priority: 100+lengthprefix
 246          * Match: prefix, l3c, "mac address of router" ?
 247          * Action:
 248          * - set Reg2, Reg3 for L3Ep by L2Ep ?
 249          * - if external,
 250          * - Reg7: use switch location external port else punt for now
 251          * - if internal
 252          * - Reg7: grab L2Ep from L3Ep and use its location info
 253          * - goto_table: POLENF (will check there for external on EP)
 254          */
 255
 256         ReadOnlyTransaction rTx = ctx.getDataBroker().newReadOnlyTransaction();
 257         // TODO Bug #3440 Target: Be - should support for more than first gateway.
 258         EndpointL3Gateways l3Gateway = prefixEp.getEndpointL3Gateways().get(0);
 259         Optional<EndpointL3> optL3Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 260                 IidFactory.l3EndpointIid(l3Gateway.getL3Context(), l3Gateway.getIpAddress()), rTx);
 261         if (!optL3Ep.isPresent()) {
 262             LOG.error("createL3PrefixFlow - L3Endpoint gateway {} for L3Prefix {} not found.", l3Gateway, prefixEp);
 263             return null;
 264         }
 265         EndpointL3 l3Ep = optL3Ep.get();
 266         if (l3Ep.getL2Context() == null || l3Ep.getMacAddress() == null) {
 267             LOG.debug("L3 endpoint representing L3 gateway does not contain L2-context or MAC address. {}", l3Ep);
 268             return null;
 269         }
 270         Optional<Endpoint> optL2Ep = readFromDs(LogicalDatastoreType.OPERATIONAL,
 271                 IidFactory.endpointIid(l3Ep.getL2Context(), l3Ep.getMacAddress()), rTx);
 272         if (!optL2Ep.isPresent()) {
 273             LOG.error("createL3PrefixFlow - L2Endpoint for L3Gateway {} not found.", l3Ep);
 274             return null;
 275         }
 276         Endpoint l2Ep = optL2Ep.get();
 277         EndpointFwdCtxOrdinals epFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, l2Ep);
 278         if (epFwdCtxOrds == null) {
 279             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", l2Ep);
 280             return null;
 281         }
 282
 283         NetworkDomainId epNetworkContainment = getEPNetworkContainment(l2Ep);
 284
 285         MacAddress epDestMac = l2Ep.getMacAddress();
 286         MacAddress destSubnetGatewayMac = l2Ep.getMacAddress();
 287         L3Context destL3c = getL3ContextForSubnet(prefixEp.getTenant(), subnet);
 288         if (destL3c == null || destL3c.getId() == null) {
 289             LOG.error("No L3 Context found associated with subnet {}", subnet.getId());
 290             return null;
 291         }
 292
 293         MacAddress matcherMac = routerPortMac(destL3c, subnet.getVirtualRouterIp());
 294
 295         ArrayList<Instruction> l3instructions = new ArrayList<>();
 296         List<Action> applyActions = new ArrayList<>();
 297         List<Action> l3ApplyActions = new ArrayList<>();
 298
 299         int order = 0;
 300
 301         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 302         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 303         Action setNextHop;
 304         String nextHop=null;
 305
 306         OfOverlayContext ofc = l2Ep.getAugmentation(OfOverlayContext.class);
 307         LocationType location;
 308
 309         if (ofc != null && ofc.getLocationType() != null) {
 310             location = ofc.getLocationType();
 311         } else if (ofc != null) {
 312             // Augmentation, but using default location
 313             location = LocationType.Internal;
 314         } else {
 315             LOG.info("createL3PrefixFlow - Endpoint {} had no augmentation.", l2Ep);
 316             return null;
 317         }
 318
 319         long portNum = -1;
 320
 321         if (location.equals(LocationType.Internal)) {
 322             checkNotNull(ofc.getNodeConnectorId());
 323             nextHop = ofc.getNodeConnectorId().getValue();
 324             try {
 325                 portNum = getOfPortNum(ofc.getNodeConnectorId());
 326             } catch (NumberFormatException ex) {
 327                 LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 328                 return null;
 329             }
 330
 331         } else {
 332             // External
 333             Set<NodeConnectorId> externalPorts = ctx.getSwitchManager().getExternalPorts(nodeId);
 334             checkNotNull(externalPorts);
 335             for (NodeConnectorId externalPort : externalPorts) {
 336                 // TODO Bug #3440 Target: Be - should support for more than first external port.
 337                 //TODO Bug 3546 - Difficult: External port is unrelated to Tenant, L3C, L2BD..
 338                 nextHop = externalPort.getValue();
 339                 try {
 340                     portNum = getOfPortNum(externalPort);
 341                 } catch (NumberFormatException ex) {
 342                     LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 343                     return null;
 344                 }
 345                 continue;
 346             }
 347         }
 348
 349         if (Strings.isNullOrEmpty(nextHop)
 350                 || portNum == -1) {
 351             LOG.error("createL3Prefix - Cannot find nodeConnectorId for {} for Prefix: ", l2Ep, prefixEp);
 352             return null;
 353         }
 354         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 355
 356         Action setDlDst = setDlDstAction(epDestMac);
 357         l3ApplyActions.add(setDlDst);
 358
 359         Action decTtl = decNwTtlAction();
 360         l3ApplyActions.add(decTtl);
 361
 362         order += 1;
 363         applyActions.add(setdEPG);
 364         applyActions.add(setdCG);
 365         applyActions.add(setNextHop);
 366
 367         applyActions.addAll(l3ApplyActions);
 368         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 369             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 370             .build();
 371
 372         l3instructions.add(applyActionsIns);
 373         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 374             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 375             .build();
 376         l3instructions.add(gotoTable);
 377
 378         Layer3Match m = null;
 379         Long etherType = null;
 380         String ikey = null;
 381         Integer prefixLength=0;
 382         if (prefixEp.getIpPrefix().getIpv4Prefix() != null) {
 383             ikey = prefixEp.getIpPrefix().getIpv4Prefix().getValue();
 384             etherType = IPv4;
 385             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv4Prefix().getValue().split("/")[1]);
 386             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 387         } else if (prefixEp.getIpPrefix().getIpv6Prefix() != null) {
 388             ikey = prefixEp.getIpPrefix().getIpv6Prefix().getValue();
 389             etherType = IPv6;
 390             /*
 391              *  This will result in flows with priority between 100-228, but since its matching on IPv6 prefix as well
 392              *  this shouldn't pose and issue, as the priority is more important within the address space of the matcher,
 393              *  even though technically flows are processed in priority order.
 394              */
 395
 396             prefixLength=Integer.valueOf(prefixEp.getIpPrefix().getIpv6Prefix().getValue().split("/")[1]);
 397             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 398         } else {
 399             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", prefixEp);
 400             return null;
 401         }
 402
 403         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType));
 404         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 405         Match match = mb.build();
 406         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "L3prefix", match);
 407         FlowBuilder flowb = base().setId(flowid)
 408             .setPriority(Integer.valueOf(BASE_L3_PRIORITY+prefixLength))
 409             .setMatch(match)
 410             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 411         return flowb.build();
 412     }
 413
 414     private Flow createBroadcastFlow(EndpointFwdCtxOrdinals epOrd) {
 415         MatchBuilder mb = new MatchBuilder()
 416                 .setEthernetMatch(new EthernetMatchBuilder().setEthernetDestination(
 417                         new EthernetDestinationBuilder().setAddress(MULTICAST_MAC)
 418                                 .setMask(MULTICAST_MAC)
 419                                 .build()).build());
 420         addNxRegMatch(mb, RegMatch.of(NxmNxReg5.class, Long.valueOf(epOrd.getFdId())));
 421
 422         Match match = mb.build();
 423         FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "broadcast", match);
 424         FlowBuilder flowb = base().setPriority(Integer.valueOf(140))
 425             .setId(flowId)
 426             .setMatch(match)
 427             .setInstructions(
 428                     instructions(applyActionIns(nxLoadTunIdAction(BigInteger.valueOf(epOrd.getFdId()), false),
 429                             groupAction(Long.valueOf(epOrd.getFdId())))));
 430
 431         return flowb.build();
 432     }
 433
 434     private MacAddress routerPortMac(L3Context l3c, IpAddress ipAddress) {
 435
 436         if (ctx.getDataBroker() == null) {
 437             return null;
 438         }
 439
 440         MacAddress defaultMacAddress = ROUTER_MAC;
 441
 442         EndpointL3Key l3Key = new EndpointL3Key(ipAddress, l3c.getId());
 443         InstanceIdentifier<EndpointL3> endpointsIid = InstanceIdentifier.builder(Endpoints.class)
 444             .child(EndpointL3.class, l3Key)
 445             .build();
 446         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
 447
 448         Optional<EndpointL3> r;
 449         try {
 450             r = t.read(LogicalDatastoreType.OPERATIONAL, endpointsIid).get();
 451             if (!r.isPresent())
 452                 return defaultMacAddress;
 453             EndpointL3 epL3 = r.get();
 454             if (epL3.getMacAddress() == null) {
 455                 return defaultMacAddress;
 456             } else {
 457                 return epL3.getMacAddress();
 458             }
 459         } catch (Exception e) {
 460             LOG.error("Error reading EndpointL3 {}.{}", l3c, ipAddress, e);
 461             return null;
 462         }
 463     }
 464
 465     private L3Context getL3ContextForSubnet(TenantId tenantId, Subnet sn) {
 466         IndexedTenant indexedTenant = ctx.getPolicyResolver().getTenant(tenantId);
 467         if (indexedTenant == null) {
 468             LOG.debug("Tenant {} is null, cannot get L3 context", tenantId);
 469             return null;
 470         }
 471         L3Context l3c = indexedTenant.resolveL3Context(sn.getId());
 472         return l3c;
 473     }
 474
 475     private Flow createRouterArpFlow(TenantId tenantId, NodeId nodeId, Subnet sn, int l3Id) {
 476         if (sn == null || sn.getVirtualRouterIp() == null) {
 477             LOG.trace("Didn't create routerArpFlow since either subnet or subnet virtual router was null");
 478             return null;
 479         }
 480         /*
 481          * TODO: Li alagalah: This should be new Yang "gateways" list as well,
 482          * that expresses the gateway and prefixes it is interface for. Should
 483          * also check for external.
 484          */
 485         if (sn.getVirtualRouterIp().getIpv4Address() != null) {
 486             String ikey = sn.getVirtualRouterIp().getIpv4Address().getValue();
 487
 488             L3Context l3c = getL3ContextForSubnet(tenantId, sn);
 489             if (l3c == null) {
 490                 LOG.error("No L3 Context found associated with subnet {}", sn.getId());
 491             }
 492
 493             MacAddress routerMac = routerPortMac(l3c, sn.getVirtualRouterIp());
 494             if (routerMac == null) {
 495                 return null;
 496             }
 497
 498             BigInteger intRouterMac = new BigInteger(1, bytesFromHexString(routerMac.getValue()));
 499
 500             MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, null, ARP)).setLayer3Match(
 501                     new ArpMatchBuilder().setArpOp(Integer.valueOf(1))
 502                         .setArpTargetTransportAddress(new Ipv4Prefix(ikey + "/32"))
 503                         .build());
 504             addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(l3Id)));
 505
 506             Match match = mb.build();
 507             FlowId flowId = FlowIdUtils.newFlowId(TABLE_ID, "routerarp", match);
 508             FlowBuilder flowb = base().setPriority(150)
 509                 .setId(flowId)
 510                 .setMatch(match)
 511                 .setInstructions(
 512                         instructions(applyActionIns(nxMoveEthSrcToEthDstAction(), setDlSrcAction(routerMac),
 513                                 nxLoadArpOpAction(BigInteger.valueOf(2L)), nxMoveArpShaToArpThaAction(),
 514                                 nxLoadArpShaAction(intRouterMac), nxMoveArpSpaToArpTpaAction(),
 515                                 nxLoadArpSpaAction(ikey), outputAction(new NodeConnectorId(nodeId.getValue()
 516                                         + ":INPORT")))));
 517             return flowb.build();
 518         } else {
 519             LOG.warn("IPv6 virtual router {} for subnet {} not supported", sn.getVirtualRouterIp(), sn.getId()
 520                 .getValue());
 521             return null;
 522         }
 523
 524     }
 525
 526     private Flow createLocalL2Flow(Endpoint ep, EndpointFwdCtxOrdinals epFwdCtxOrds, OfOverlayContext ofc) {
 527
 528         // TODO Li alagalah - refactor common code but keep simple method
 529         ArrayList<Instruction> instructions = new ArrayList<>();
 530         List<Action> applyActions = new ArrayList<>();
 531
 532         int order = 0;
 533
 534         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 535         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 536         Action setNextHop;
 537         String nextHop;
 538
 539         // BEGIN L2 LOCAL
 540         nextHop = ofc.getNodeConnectorId().getValue();
 541
 542         long portNum;
 543         try {
 544             portNum = getOfPortNum(ofc.getNodeConnectorId());
 545         } catch (NumberFormatException ex) {
 546             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 547             return null;
 548         }
 549
 550         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 551
 552         // END L2 LOCAL
 553
 554         order += 1;
 555         applyActions.add(setdEPG);
 556         applyActions.add(setdCG);
 557         applyActions.add(setNextHop);
 558         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 559             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 560             .build();
 561         instructions.add(applyActionsIns);
 562
 563         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 564             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 565             .build();
 566         instructions.add(gotoTable);
 567
 568         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 569         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(epFwdCtxOrds.getBdId())));
 570         Match match = mb.build();
 571         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL2", match);
 572         FlowBuilder flowb = base().setId(flowid)
 573             .setPriority(Integer.valueOf(50))
 574             .setMatch(match)
 575             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 576         return flowb.build();
 577     }
 578
 579     private void syncEP(OfWriter ofWriter, NodeId nodeId, PolicyInfo policyInfo, Endpoint srcEp, Endpoint destEp)
 580             throws Exception {
 581
 582         if (ctx.getPolicyResolver().getTenant(srcEp.getTenant()) == null
 583                 || ctx.getPolicyResolver().getTenant(destEp.getTenant()) == null) {
 584             LOG.debug("Source or destination EP references empty tenant srcEp:{} destEp:{}", srcEp, destEp);
 585             return;
 586         }
 587
 588         // TODO: Conditions messed up, but for now, send policyInfo until this
 589         // is fixed.
 590         EndpointFwdCtxOrdinals destEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, destEp);
 591         if (destEpFwdCtxOrds == null) {
 592             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", destEp);
 593             return;
 594         }
 595         EndpointFwdCtxOrdinals srcEpFwdCtxOrds = OrdinalFactory.getEndpointFwdCtxOrdinals(ctx, policyInfo, srcEp);
 596         if (srcEpFwdCtxOrds == null) {
 597             LOG.debug("getEndpointFwdCtxOrdinals is null for EP {}", srcEp);
 598             return;
 599         }
 600
 601
 602         if (destEp.getTenant() == null || (destEp.getEndpointGroup() == null && destEp.getEndpointGroups() == null)) {
 603             if (destEp.getTenant() == null) {
 604                 LOG.debug("Didn't process endpoint {} due to tenant being null", destEp.getKey());
 605             } else {
 606                 LOG.debug("Didn't process endpoint {} due to EPG(s) being null", destEp.getKey());
 607             }
 608             return;
 609         }
 610         OfOverlayContext ofc = destEp.getAugmentation(OfOverlayContext.class);
 611
 612         if (LocationType.External.equals(ofc.getLocationType())) {
 613             LOG.error("syncEp(): External endpoints should not be seen here.");
 614             return;
 615         }
 616
 617         /*
 618          * Only care about subnets for L3, but fetch them before loop. We need
 619          * the local subnets for setting SRC MAC for routing. All Routing is now
 620          * done locally! YAY! Instead of being shovelled L2 style across network
 621          * ala Helium.
 622          */
 623         List<Subnet> localSubnets = getLocalSubnets(nodeId);
 624         if (localSubnets == null) {
 625             LOG.error("No subnets could be found locally for node: {}", nodeId);
 626             return;
 627         }
 628
 629         if (Objects.equals(ofc.getNodeId(), nodeId)) {
 630             // this is a local endpoint; send to the approppriate local
 631             // port
 632
 633             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 634                 ofWriter.writeFlow(nodeId, TABLE_ID, createLocalL2Flow(destEp, destEpFwdCtxOrds, ofc));
 635             }
 636             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 637             // The Endpoint conflation must end!
 638             if (destEp.getL3Address() == null) {
 639                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 640                 return;
 641             }
 642
 643             for (L3Address l3a : destEp.getL3Address()) {
 644                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 645                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 646                             destEp.getL3Address());
 647                     continue;
 648                 } else {
 649                     for (Subnet localSubnet : localSubnets) {
 650                         Flow flow = createLocalL3RoutedFlow(destEp, l3a, destEpFwdCtxOrds, ofc, localSubnet);
 651                         if (flow != null) {
 652                             ofWriter.writeFlow(nodeId, TABLE_ID, flow);
 653                         } else {
 654                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 655                                     localSubnet.getIpPrefix().getValue());
 656                         }
 657                     }
 658                 }
 659             }
 660         } else {
 661             // this endpoint is on a different switch; send to the
 662             // appropriate tunnel
 663             if (srcEpFwdCtxOrds.getBdId() == destEpFwdCtxOrds.getBdId()) {
 664                 Flow remoteL2Flow = createRemoteL2Flow(destEp, nodeId, srcEpFwdCtxOrds, destEpFwdCtxOrds, ofc);
 665                 if (remoteL2Flow != null) {
 666                     ofWriter.writeFlow(nodeId, TABLE_ID, remoteL2Flow);
 667                 }
 668             } else {
 669                 LOG.trace("DestinationMapper: RemoteL2Flow: not created, in different BDs src: {} dst: {}",
 670                         srcEpFwdCtxOrds.getBdId(), destEpFwdCtxOrds.getBdId());
 671             }
 672
 673             // TODO Li alagalah: Need to move to EndpointL3 for L3 processing.
 674             // The Endpoint conflation must end!
 675             if (destEp.getL3Address() == null) {
 676                 LOG.trace("Endpoint {} didn't have L3 Address so was not processed for L3 flows.", destEp.getKey());
 677                 return;
 678             }
 679             for (L3Address l3a : destEp.getL3Address()) {
 680                 if (l3a.getIpAddress() == null || l3a.getL3Context() == null) {
 681                     LOG.error("Endpoint with L3Address but either IPAddress or L3Context is null. {}",
 682                             destEp.getL3Address());
 683                     continue;
 684                 } else {
 685                     for (Subnet localSubnet : localSubnets) {
 686                         Flow remoteL3Flow = createRemoteL3RoutedFlow(destEp, l3a, nodeId, srcEpFwdCtxOrds,
 687                                 destEpFwdCtxOrds, ofc, localSubnet);
 688                         if (remoteL3Flow != null) {
 689                             ofWriter.writeFlow(nodeId, TABLE_ID, remoteL3Flow);
 690                         } else {
 691                             LOG.trace("Did not write remote L3 flow for endpoint {} and subnet {}", l3a.getIpAddress(),
 692                                     localSubnet.getIpPrefix().getValue());
 693                         }
 694                     }
 695                 }
 696             }
 697         } // remote (tunnel)
 698
 699         // }
 700
 701     }
 702
 703     /*
 704      * ################################## DestMapper Flow methods
 705      * ##################################
 706      */
 707     private Flow createLocalL3RoutedFlow(Endpoint destEp, L3Address destL3Address, EndpointFwdCtxOrdinals epFwdCtxOrds,
 708             OfOverlayContext ofc, Subnet srcSubnet) {
 709
 710         // TODO Li alagalah - refactor common code but keep simple method
 711
 712         Subnet destSubnet = null;
 713         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 714         if (subnets == null) {
 715             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 716             return null;
 717         }
 718         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 719         for (Subnet subnet : subnets) {
 720             // TODO Li alagalah add IPv6 support
 721             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 722                 destSubnet = subnet;
 723                 break;
 724             }
 725         }
 726         if (destSubnet == null) {
 727             LOG.trace("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 728             return null;
 729         }
 730
 731         if (destSubnet.getVirtualRouterIp() == null) {
 732             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 733                     destL3Address.getKey());
 734             return null;
 735         }
 736
 737         if (srcSubnet.getVirtualRouterIp() == null) {
 738             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 739             return null;
 740         }
 741         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 742         if (destL3c == null || destL3c.getId() == null) {
 743             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 744             return null;
 745         }
 746         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 747         if (srcL3c == null || srcL3c.getId() == null) {
 748             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 749             return null;
 750         }
 751
 752         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 753             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 754                 .getValue(), destL3c.getId().getValue());
 755             return null;
 756         }
 757
 758         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 759         MacAddress epDestMac = destEp.getMacAddress();
 760         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 761
 762         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 763             // This is our final destination, so match on actual EP mac.
 764             matcherMac = epDestMac;
 765         }
 766
 767         ArrayList<Instruction> l3instructions = new ArrayList<>();
 768         List<Action> applyActions = new ArrayList<>();
 769         List<Action> l3ApplyActions = new ArrayList<>();
 770
 771         int order = 0;
 772
 773         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(epFwdCtxOrds.getEpgId()));
 774         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(epFwdCtxOrds.getCgId()));
 775         Action setNextHop;
 776         String nextHop;
 777
 778         // BEGIN L3 LOCAL
 779         nextHop = ofc.getNodeConnectorId().getValue();
 780
 781         long portNum;
 782         try {
 783             portNum = getOfPortNum(ofc.getNodeConnectorId());
 784         } catch (NumberFormatException ex) {
 785             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 786             return null;
 787         }
 788
 789         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 790         // END L3 LOCAL
 791
 792         // Lets not re-write the srcMac if its local.
 793         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
 794             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
 795             l3ApplyActions.add(setDlSrc);
 796         }
 797
 798         Action setDlDst = setDlDstAction(epDestMac);
 799         l3ApplyActions.add(setDlDst);
 800
 801         Action decTtl = decNwTtlAction();
 802         l3ApplyActions.add(decTtl);
 803
 804         order += 1;
 805         applyActions.add(setdEPG);
 806         applyActions.add(setdCG);
 807         applyActions.add(setNextHop);
 808
 809         applyActions.addAll(l3ApplyActions);
 810         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 811             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 812             .build();
 813
 814         l3instructions.add(applyActionsIns);
 815         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 816             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 817             .build();
 818         l3instructions.add(gotoTable);
 819         Layer3Match m = null;
 820         Long etherType = null;
 821         String ikey = null;
 822         if (destL3Address.getIpAddress().getIpv4Address() != null) {
 823             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
 824             etherType = IPv4;
 825             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
 826         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
 827             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
 828             etherType = IPv6;
 829             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
 830         } else {
 831             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
 832             return null;
 833         }
 834
 835         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
 836             .setLayer3Match(m);
 837         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(epFwdCtxOrds.getL3Id())));
 838         Match match = mb.build();
 839         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "localL3", match);
 840         FlowBuilder flowb = base().setId(flowid)
 841             .setPriority(Integer.valueOf(132))
 842             .setMatch(match)
 843             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
 844         return flowb.build();
 845     }
 846
 847     private Flow createRemoteL2Flow(Endpoint ep, NodeId nodeId, EndpointFwdCtxOrdinals srcEpFwdCtxOrds,
 848             EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc) {
 849
 850         // TODO Li alagalah - refactor common code but keep simple method
 851
 852         // this endpoint is on a different switch; send to the
 853         // appropriate tunnel
 854
 855         ArrayList<Instruction> instructions = new ArrayList<>();
 856         List<Action> applyActions = new ArrayList<>();
 857
 858         int order = 0;
 859
 860         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
 861         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
 862         Action setNextHop;
 863         String nextHop;
 864
 865         // BEGIN TUNNEL HANDLING
 866         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
 867         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
 868         if (tunDst == null) {
 869             LOG.warn("Failed to get Tunnel IP for NodeId {} with EP {}", nodeId, ep);
 870             return null;
 871         }
 872         if (tunPort == null) {
 873             LOG.warn("Failed to get Tunnel Port for NodeId {} with EP {}", nodeId, ep);
 874             return null;
 875         }
 876
 877         Action tundstAction;
 878
 879         if (tunDst.getIpv4Address() != null) {
 880             nextHop = tunDst.getIpv4Address().getValue();
 881             tundstAction = nxLoadTunIPv4Action(nextHop, false);
 882         } else if (tunDst.getIpv6Address() != null) {
 883             // nextHop = tunDst.getIpv6Address().getValue();
 884             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
 885                     ofc.getNodeId());
 886             return null;
 887         } else {
 888             // this shouldn't happen
 889             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
 890             return null;
 891         }
 892
 893         long portNum;
 894         try {
 895             portNum = getOfPortNum(tunPort);
 896         } catch (NumberFormatException ex) {
 897             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
 898             return null;
 899         }
 900
 901         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
 902         applyActions.add(tundstAction);
 903         // END TUNNEL
 904
 905         order += 1;
 906         applyActions.add(setdEPG);
 907         applyActions.add(setdCG);
 908         applyActions.add(setNextHop);
 909         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
 910             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 911             .build();
 912         instructions.add(applyActionsIns);
 913
 914         applyActionsIns = new InstructionBuilder().setOrder(order++)
 915             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
 916             .build();
 917
 918         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
 919             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
 920             .build();
 921         instructions.add(gotoTable);
 922
 923         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, ep.getMacAddress(), null));
 924         addNxRegMatch(mb, RegMatch.of(NxmNxReg4.class, Long.valueOf(destEpFwdCtxOrds.getBdId())));
 925         Match match = mb.build();
 926         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL2", match);
 927         FlowBuilder flowb = base().setId(flowid)
 928             .setPriority(Integer.valueOf(50))
 929             .setMatch(match)
 930             .setInstructions(new InstructionsBuilder().setInstruction(instructions).build());
 931
 932         return flowb.build();
 933     }
 934
 935     private Flow createRemoteL3RoutedFlow(Endpoint destEp, L3Address destL3Address, NodeId nodeId,
 936             EndpointFwdCtxOrdinals srcEpFwdCtxOrds, EndpointFwdCtxOrdinals destEpFwdCtxOrds, OfOverlayContext ofc,
 937             Subnet srcSubnet) {
 938
 939         // TODO Li alagalah - refactor common code but keep simple method
 940
 941         // this endpoint is on a different switch; send to the
 942         // appropriate tunnel
 943         Subnet destSubnet = null;
 944         HashSet<Subnet> subnets = getSubnets(destEp.getTenant());
 945         if (subnets == null) {
 946             LOG.trace("No subnets in tenant {}", destEp.getTenant());
 947             return null;
 948         }
 949         NetworkDomainId epNetworkContainment = getEPNetworkContainment(destEp);
 950         for (Subnet subnet : subnets) {
 951             // TODO Li alagalah add IPv6 support
 952             if (subnet.getId().getValue().equals(epNetworkContainment.getValue())) {
 953                 destSubnet = subnet;
 954                 break;
 955             }
 956         }
 957         if (destSubnet == null) {
 958             LOG.info("Destination IP address does not match any subnet in tenant {}", destL3Address.getIpAddress());
 959             return null;
 960         }
 961
 962         if (destSubnet.getVirtualRouterIp() == null) {
 963             LOG.trace("Destination subnet {} for Endpoint {}.{} has no gateway IP", destSubnet.getIpPrefix(),
 964                     destL3Address.getKey());
 965             return null;
 966         }
 967
 968         if (srcSubnet.getVirtualRouterIp() == null) {
 969             LOG.trace("Local subnet {} has no gateway IP", srcSubnet.getIpPrefix());
 970             return null;
 971         }
 972         L3Context destL3c = getL3ContextForSubnet(destEp.getTenant(), destSubnet);
 973         if (destL3c == null || destL3c.getId() == null) {
 974             LOG.error("No L3 Context found associated with subnet {}", destSubnet.getId());
 975             return null;
 976         }
 977         L3Context srcL3c = getL3ContextForSubnet(destEp.getTenant(), srcSubnet);
 978         if (srcL3c == null || srcL3c.getId() == null) {
 979             LOG.error("No L3 Context found associated with subnet {}", srcSubnet.getId());
 980             return null;
 981         }
 982
 983         if (!(srcL3c.getId().getValue().equals(destL3c.getId().getValue()))) {
 984             LOG.trace("Trying to route between two L3Contexts {} and {}. Not currently supported.", srcL3c.getId()
 985                 .getValue(), destL3c.getId().getValue());
 986             return null;
 987         }
 988
 989         MacAddress matcherMac = routerPortMac(destL3c, srcSubnet.getVirtualRouterIp());
 990         MacAddress epDestMac = destEp.getMacAddress();
 991         MacAddress destSubnetGatewayMac = routerPortMac(destL3c, destSubnet.getVirtualRouterIp());
 992         if (srcSubnet.getId().getValue().equals(destSubnet.getId().getValue())) {
 993             // This is our final destination, so match on actual EP mac.
 994             matcherMac = epDestMac;
 995         }
 996         ArrayList<Instruction> l3instructions = new ArrayList<>();
 997         List<Action> applyActions = new ArrayList<>();
 998         List<Action> l3ApplyActions = new ArrayList<>();
 999
1000         int order = 0;
1001
1002         Action setdEPG = nxLoadRegAction(NxmNxReg2.class, BigInteger.valueOf(destEpFwdCtxOrds.getEpgId()));
1003         Action setdCG = nxLoadRegAction(NxmNxReg3.class, BigInteger.valueOf(destEpFwdCtxOrds.getCgId()));
1004         Action setNextHop;
1005         String nextHop;
1006
1007         // BEGIN TUNNEL HANDLING
1008         IpAddress tunDst = ctx.getSwitchManager().getTunnelIP(ofc.getNodeId(), TunnelTypeVxlan.class);
1009         NodeConnectorId tunPort = ctx.getSwitchManager().getTunnelPort(nodeId, TunnelTypeVxlan.class);
1010         if (tunDst == null) {
1011             LOG.warn("Failed to get Tunnel IP for NodeId {} with L3Address {}", nodeId, destL3Address);
1012             return null;
1013         }
1014         if (tunPort == null) {
1015             LOG.warn("Failed to get Tunnel port for NodeId {} with L3Address {}", nodeId, destL3Address);
1016             return null;
1017         }
1018
1019         Action tundstAction;
1020
1021         if (tunDst.getIpv4Address() != null) {
1022             nextHop = tunDst.getIpv4Address().getValue();
1023             tundstAction = nxLoadTunIPv4Action(nextHop, false);
1024         } else if (tunDst.getIpv6Address() != null) {
1025             // nextHop = tunDst.getIpv6Address().getValue();
1026             LOG.error("IPv6 tunnel destination {} for {} not supported", tunDst.getIpv6Address().getValue(),
1027                     ofc.getNodeId());
1028             return null;
1029         } else {
1030             // this shouldn't happen
1031             LOG.error("Tunnel IP for {} invalid", ofc.getNodeId());
1032             return null;
1033         }
1034
1035         long portNum;
1036         try {
1037             portNum = getOfPortNum(tunPort);
1038         } catch (NumberFormatException ex) {
1039             LOG.warn("Could not parse port number {}", ofc.getNodeConnectorId(), ex);
1040             return null;
1041         }
1042
1043         setNextHop = nxLoadRegAction(NxmNxReg7.class, BigInteger.valueOf(portNum));
1044         applyActions.add(tundstAction);
1045         // END TUNNEL
1046
1047         order += 1;
1048         applyActions.add(setdEPG);
1049         applyActions.add(setdCG);
1050         applyActions.add(setNextHop);
1051
1052         // Lets not re-write the srcMac if its local.
1053         if (!(matcherMac.getValue().equals(epDestMac.getValue()))) {
1054             Action setDlSrc = setDlSrcAction(destSubnetGatewayMac);
1055             l3ApplyActions.add(setDlSrc);
1056         }
1057
1058         Action setDlDst = setDlDstAction(epDestMac);
1059         l3ApplyActions.add(setDlDst);
1060
1061         Action decTtl = decNwTtlAction();
1062         l3ApplyActions.add(decTtl);
1063
1064         applyActions.addAll(l3ApplyActions);
1065         Instruction applyActionsIns = new InstructionBuilder().setOrder(order++)
1066             .setInstruction(applyActionIns(applyActions.toArray(new Action[applyActions.size()])))
1067             .build();
1068
1069         l3instructions.add(applyActionsIns);
1070         Instruction gotoTable = new InstructionBuilder().setOrder(order++)
1071             .setInstruction(gotoTableIns(ctx.getPolicyManager().getTABLEID_POLICY_ENFORCER()))
1072             .build();
1073         l3instructions.add(gotoTable);
1074         Layer3Match m = null;
1075         Long etherType = null;
1076         String ikey = null;
1077         if (destL3Address.getIpAddress().getIpv4Address() != null) {
1078             ikey = destL3Address.getIpAddress().getIpv4Address().getValue() + "/32";
1079             etherType = IPv4;
1080             m = new Ipv4MatchBuilder().setIpv4Destination(new Ipv4Prefix(ikey)).build();
1081         } else if (destL3Address.getIpAddress().getIpv6Address() != null) {
1082             ikey = destL3Address.getIpAddress().getIpv6Address().getValue() + "/128";
1083             etherType = IPv6;
1084             m = new Ipv6MatchBuilder().setIpv6Destination(new Ipv6Prefix(ikey)).build();
1085         } else {
1086             LOG.error("Endpoint has IPAddress that is not recognised as either IPv4 or IPv6.", destL3Address.toString());
1087             return null;
1088         }
1089
1090         MatchBuilder mb = new MatchBuilder().setEthernetMatch(ethernetMatch(null, matcherMac, etherType))
1091             .setLayer3Match(m);
1092         addNxRegMatch(mb, RegMatch.of(NxmNxReg6.class, Long.valueOf(destEpFwdCtxOrds.getL3Id())));
1093         Match match = mb.build();
1094         FlowId flowid = FlowIdUtils.newFlowId(TABLE_ID, "remoteL3", match);
1095         FlowBuilder flowb = base().setId(flowid)
1096             .setPriority(Integer.valueOf(132))
1097             .setMatch(match)
1098             .setInstructions(new InstructionsBuilder().setInstruction(l3instructions).build());
1099         return flowb.build();
1100     }
1101
1102     private NetworkDomainId getEPNetworkContainment(Endpoint endpoint) {
1103         if (endpoint.getNetworkContainment() != null) {
1104             return endpoint.getNetworkContainment();
1105         } else {
1106             /*
1107              * TODO: Be alagalah: Endpoint Refactor: This should be set on input
1108              * which we can't do because of the backwards way endpoints were
1109              * "architected".
1110              */
1111             return ctx.getPolicyResolver()
1112                 .getTenant(endpoint.getTenant())
1113                 .getEndpointGroup(endpoint.getEndpointGroup())
1114                 .getNetworkDomain();
1115         }
1116     }
1117
1118     private HashSet<Subnet> getSubnets(final TenantId tenantId) {
1119
1120         // if (subnetsByTenant.get(tenantId) != null) {
1121         // return subnetsByTenant.get(tenantId);
1122         // }
1123
1124         if (ctx.getDataBroker() == null) {
1125             return null;
1126         }
1127
1128         ReadOnlyTransaction t = ctx.getDataBroker().newReadOnlyTransaction();
1129         InstanceIdentifier<Tenant> tiid = TenantUtils.tenantIid(tenantId);
1130         Optional<Tenant> tenantInfo;
1131         try {
1132             tenantInfo = t.read(LogicalDatastoreType.CONFIGURATION, tiid).get();
1133         } catch (Exception e) {
1134             LOG.error("Could not read Tenant {}", tenantId, e);
1135             return null;
1136         }
1137
1138         HashSet<Subnet> subnets = new HashSet<Subnet>();
1139
1140         if (!tenantInfo.isPresent()) {
1141             LOG.warn("Tenant {} not found", tenantId);
1142             return null;
1143         }
1144
1145         subnets.addAll(tenantInfo.get().getSubnet());
1146         // subnetsByTenant.put(tenantId, subnets);
1147         return subnets;
1148     }
1149
1150     // Need a method to get subnets for EPs attached to the node locally
1151     // to set the source Mac address for the router interface.
1152     private List<Subnet> getLocalSubnets(NodeId nodeId) {
1153         Collection<Endpoint> endpointsForNode = ctx.getEndpointManager().getEndpointsForNode(nodeId);
1154
1155         List<Subnet> localSubnets = new ArrayList<Subnet>();
1156
1157         for (Endpoint endpoint : endpointsForNode) {
1158             HashSet<Subnet> subnets = getSubnets(endpoint.getTenant());
1159             if (subnets == null) {
1160                 LOG.debug("No local subnets in tenant {} for EP {}.", endpoint.getTenant(), endpoint.getKey());
1161                 continue;
1162             }
1163             NetworkDomainId epNetworkContainment = getEPNetworkContainment(endpoint);
1164             for (Subnet subnet : subnets) {
1165                 if (epNetworkContainment.getValue().equals(subnet.getId().getValue())) {
1166                     localSubnets.add(subnet);
1167                 }
1168             }
1169         }
1170         return localSubnets;
1171     }
1172
1173     static byte[] bytesFromHexString(String values) {
1174         String target = "";
1175         if (values != null) {
1176             target = values;
1177         }
1178         String[] octets = target.split(":");
1179
1180         byte[] ret = new byte[octets.length];
1181         for (int i = 0; i < octets.length; i++) {
1182             ret[i] = Integer.valueOf(octets[i], 16).byteValue();
1183         }
1184         return ret;
1185     }
1186 }