openstack/net-virt-providers/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/providers/openflow13/OF13Provider.java

   1 /*
   2  * Copyright (c) 2013, 2015 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.providers.openflow13;
  10
  11 import java.net.InetAddress;
  12 import java.util.List;
  13 import java.util.Map;
  14 import java.util.concurrent.ExecutionException;
  15
  16 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  17 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  18 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  19 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  20 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  21 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  22 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork;
  23 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronPort;
  24 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSecurityGroup;
  25 import org.opendaylight.ovsdb.openstack.netvirt.translator.Neutron_IPs;
  26 import org.opendaylight.ovsdb.openstack.netvirt.MdsalHelper;
  27 import org.opendaylight.ovsdb.openstack.netvirt.NetworkHandler;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.BridgeConfigurationManager;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.ClassifierProvider;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService;
  31 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  32 import org.opendaylight.ovsdb.openstack.netvirt.api.EgressAclProvider;
  33 import org.opendaylight.ovsdb.openstack.netvirt.api.IngressAclProvider;
  34 import org.opendaylight.ovsdb.openstack.netvirt.api.L2ForwardingProvider;
  35 import org.opendaylight.ovsdb.openstack.netvirt.api.NetworkingProvider;
  36 import org.opendaylight.ovsdb.openstack.netvirt.api.NetworkingProviderManager;
  37 import org.opendaylight.ovsdb.openstack.netvirt.api.NodeCacheManager;
  38 import org.opendaylight.ovsdb.openstack.netvirt.api.SecurityServicesManager;
  39 import org.opendaylight.ovsdb.openstack.netvirt.api.Southbound;
  40 import org.opendaylight.ovsdb.openstack.netvirt.api.Status;
  41 import org.opendaylight.ovsdb.openstack.netvirt.api.StatusCode;
  42 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  43 import org.opendaylight.ovsdb.openstack.netvirt.providers.ConfigInterface;
  44 import org.opendaylight.ovsdb.openstack.netvirt.providers.NetvirtProvidersProvider;
  45 import org.opendaylight.ovsdb.utils.mdsal.openflow.FlowUtils;
  46 import org.opendaylight.ovsdb.utils.mdsal.openflow.InstructionUtils;
  47 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  48 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Uri;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.GroupActionCase;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.GroupActionCaseBuilder;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCaseBuilder;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.group.action._case.GroupActionBuilder;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.output.action._case.OutputActionBuilder;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.ActionBuilder;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.ActionKey;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.InstructionsBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.flow.MatchBuilder;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.ApplyActionsCase;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.ApplyActionsCaseBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.instruction.apply.actions._case.ApplyActionsBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionKey;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.BucketId;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.Buckets;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.BucketsBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.BucketBuilder;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.BucketKey;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.GroupBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.GroupKey;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  90 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  91 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  92 import org.osgi.framework.BundleContext;
  93 import org.osgi.framework.ServiceReference;
  94 import org.slf4j.Logger;
  95 import org.slf4j.LoggerFactory;
  96
  97 import com.google.common.base.Optional;
  98 import com.google.common.base.Preconditions;
  99 import com.google.common.collect.Lists;
 100 import com.google.common.collect.Maps;
 101 import com.google.common.util.concurrent.CheckedFuture;
 102
 103
 104 /**
 105  * Open vSwitch OpenFlow 1.3 Networking Provider for OpenStack Neutron
 106  *
 107  * @author Madhu Venugopal
 108  * @author Brent Salisbury
 109  * @author Dave Tucker
 110  * @author Sam Hague
 111  */
 112 // Methods' parameters in this class follow the same pattern to avoid confusion between same-typed parameters
 113 // The patterns need to be preserved even though not all parameters are used in all methods
 114 @SuppressWarnings("UnusedParameters")
 115 public class OF13Provider implements ConfigInterface, NetworkingProvider {
 116     private static final Logger LOG = LoggerFactory.getLogger(OF13Provider.class);
 117     private static final short TABLE_0_DEFAULT_INGRESS = 0;
 118     private static final short TABLE_1_ISOLATE_TENANT = 10;
 119     private static final short TABLE_2_LOCAL_FORWARD = 20;
 120     private static Long groupId = 1L;
 121     private DataBroker dataBroker = null;
 122
 123     private volatile ConfigurationService configurationService;
 124     private volatile BridgeConfigurationManager bridgeConfigurationManager;
 125     private volatile TenantNetworkManager tenantNetworkManager;
 126     private volatile SecurityServicesManager securityServicesManager;
 127     private volatile ClassifierProvider classifierProvider;
 128     private volatile IngressAclProvider ingressAclProvider;
 129     private volatile EgressAclProvider egressAclProvider;
 130     private volatile NodeCacheManager nodeCacheManager;
 131     private volatile L2ForwardingProvider l2ForwardingProvider;
 132
 133     public static final String NAME = "OF13Provider";
 134     private volatile BundleContext bundleContext;
 135     private volatile Southbound southbound;
 136
 137     public OF13Provider() {
 138         this.dataBroker = NetvirtProvidersProvider.getDataBroker();
 139     }
 140
 141     @Override
 142     public String getName() {
 143         return NAME;
 144     }
 145
 146     @Override
 147     public boolean supportsServices() {
 148         return true;
 149     }
 150
 151     @Override
 152     public boolean hasPerTenantTunneling() {
 153         return false;
 154     }
 155
 156     // The method is tested for in OF13ProviderTest
 157     @SuppressWarnings("unused")
 158     private Status getTunnelReadinessStatus (Node node, String tunnelKey) {
 159         InetAddress srcTunnelEndPoint = configurationService.getTunnelEndPoint(node);
 160         if (srcTunnelEndPoint == null) {
 161             LOG.error("Tunnel Endpoint not configured for Node {}", node);
 162             return new Status(StatusCode.NOTFOUND, "Tunnel Endpoint not configured for "+ node);
 163         }
 164
 165         if (!bridgeConfigurationManager.isNodeNeutronReady(node)) {
 166             LOG.error("{} is not Overlay ready", node);
 167             return new Status(StatusCode.NOTACCEPTABLE, node+" is not Overlay ready");
 168         }
 169
 170         if (!tenantNetworkManager.isTenantNetworkPresentInNode(node, tunnelKey)) {
 171             LOG.debug("{} has no VM corresponding to segment {}", node, tunnelKey);
 172             return new Status(StatusCode.NOTACCEPTABLE, node+" has no VM corresponding to segment "+ tunnelKey);
 173         }
 174         return new Status(StatusCode.SUCCESS);
 175     }
 176
 177     private String getTunnelName(String tunnelType, InetAddress dst) {
 178         return tunnelType+"-"+dst.getHostAddress();
 179     }
 180
 181     private boolean addTunnelPort (Node node, String tunnelType, InetAddress src, InetAddress dst) {
 182         String tunnelBridgeName = configurationService.getIntegrationBridgeName();
 183         String portName = getTunnelName(tunnelType, dst);
 184         LOG.info("addTunnelPort enter: portName: {}", portName);
 185         if (southbound.extractTerminationPointAugmentation(node, portName) != null
 186                 || southbound.isTunnelTerminationPointExist(node, tunnelBridgeName, portName)) {
 187             LOG.info("Tunnel {} is present in {} of {}", portName, tunnelBridgeName, node.getNodeId().getValue());
 188             return true;
 189         }
 190
 191         Map<String, String> options = Maps.newHashMap();
 192         options.put("key", "flow");
 193         options.put("local_ip", src.getHostAddress());
 194         options.put("remote_ip", dst.getHostAddress());
 195
 196         if (!southbound.addTunnelTerminationPoint(node, tunnelBridgeName, portName, tunnelType, options)) {
 197             LOG.error("Failed to insert Tunnel port {} in {}", portName, tunnelBridgeName);
 198             return false;
 199         }
 200
 201             LOG.info("addTunnelPort exit: portName: {}", portName);
 202         return true;
 203     }
 204
 205     /* delete port from ovsdb port table */
 206     private boolean deletePort(Node node, String bridgeName, String portName) {
 207         // TODO SB_MIGRATION
 208         // might need to convert from ovsdb node to bridge node
 209         return southbound.deleteTerminationPoint(node, portName);
 210     }
 211
 212     private boolean deleteTunnelPort(Node node, String tunnelType, InetAddress src, InetAddress dst) {
 213         String tunnelBridgeName = configurationService.getIntegrationBridgeName();
 214         String portName = getTunnelName(tunnelType, dst);
 215         return deletePort(node, tunnelBridgeName, portName);
 216     }
 217
 218     private boolean deletePhysicalPort(Node node, String phyIntfName) {
 219         String intBridgeName = configurationService.getIntegrationBridgeName();
 220         return deletePort(node, intBridgeName, phyIntfName);
 221     }
 222
 223     private void programLocalBridgeRules(Node node, Long dpid, String segmentationId,
 224                                          String attachedMac, long localPort) {
 225         /*
 226          * Table(0) Rule #3
 227          * ----------------
 228          * Match: VM sMac and Local Ingress Port
 229          * Action:Action: Set Tunnel ID and GOTO Local Table (5)
 230          */
 231
 232         handleLocalInPort(dpid, TABLE_0_DEFAULT_INGRESS, TABLE_1_ISOLATE_TENANT,
 233                 segmentationId, localPort, attachedMac, true);
 234
 235         /*
 236          * Table(0) Rule #4
 237          * ----------------
 238          * Match: Drop any remaining Ingress Local VM Packets
 239          * Action: Drop w/ a low priority
 240          */
 241
 242         handleDropSrcIface(dpid, localPort, true);
 243
 244         /*
 245          * Table(2) Rule #1
 246          * ----------------
 247          * Match: Match TunID and Destination DL/dMAC Addr
 248          * Action: Output Port
 249          * table=2,tun_id=0x5,dl_dst=00:00:00:00:00:01 actions=output:2
 250          */
 251
 252         handleLocalUcastOut(dpid, TABLE_2_LOCAL_FORWARD, segmentationId, localPort, attachedMac, true);
 253
 254         /*
 255          * Table(2) Rule #2
 256          * ----------------
 257          * Match: Tunnel ID and dMAC (::::FF:FF)
 258          * table=2,priority=16384,tun_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
 259          * actions=output:2,3,4,5
 260          */
 261
 262         handleLocalBcastOut(dpid, TABLE_2_LOCAL_FORWARD, segmentationId, localPort, true);
 263         handleTunnelFloodOut(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD, segmentationId, localPort, true);
 264
 265         /*
 266          * TODO : Optimize the following 2 writes to be restricted only for the very first port known in a segment.
 267          */
 268         /*
 269          * Table(1) Rule #3
 270          * ----------------
 271          * Match:  Any remaining Ingress Local VM Packets
 272          * Action: Drop w/ a low priority
 273          * -------------------------------------------
 274          * table=1,priority=8192,tun_id=0x5 actions=goto_table:2
 275          */
 276
 277         handleTunnelMiss(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD, segmentationId, true);
 278
 279         /*
 280          * Table(2) Rule #3
 281          * ----------------
 282          * Match: Any Remaining Flows w/a TunID
 283          * Action: Drop w/ a low priority
 284          * table=2,priority=8192,tun_id=0x5 actions=drop
 285          */
 286
 287         handleLocalTableMiss(dpid, TABLE_2_LOCAL_FORWARD, segmentationId, true);
 288     }
 289
 290     private void removeLocalBridgeRules(Node node, Long dpid, String segmentationId, String attachedMac, long localPort) {
 291         /*
 292          * Table(0) Rule #3
 293          * ----------------
 294          * Match: VM sMac and Local Ingress Port
 295          * Action:Action: Set Tunnel ID and GOTO Local Table (5)
 296          */
 297
 298         handleLocalInPort(dpid, TABLE_0_DEFAULT_INGRESS, TABLE_1_ISOLATE_TENANT, segmentationId, localPort, attachedMac, false);
 299
 300         /*
 301          * Table(0) Rule #4
 302          * ----------------
 303          * Match: Drop any remaining Ingress Local VM Packets
 304          * Action: Drop w/ a low priority
 305          */
 306
 307         handleDropSrcIface(dpid, localPort, false);
 308
 309         /*
 310          * Table(2) Rule #1
 311          * ----------------
 312          * Match: Match TunID and Destination DL/dMAC Addr
 313          * Action: Output Port
 314          * table=2,tun_id=0x5,dl_dst=00:00:00:00:00:01 actions=output:2
 315          */
 316
 317         handleLocalUcastOut(dpid, TABLE_2_LOCAL_FORWARD, segmentationId, localPort, attachedMac, false);
 318
 319         /*
 320          * Table(2) Rule #2
 321          * ----------------
 322          * Match: Tunnel ID and dMAC (::::FF:FF)
 323          * table=2,priority=16384,tun_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
 324          * actions=output:2,3,4,5
 325          */
 326
 327         handleLocalBcastOut(dpid, TABLE_2_LOCAL_FORWARD, segmentationId, localPort, false);
 328         handleTunnelFloodOut(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD, segmentationId, localPort, false);
 329     }
 330
 331     private void programLocalIngressTunnelBridgeRules(Node node, Long dpid, String segmentationId, String attachedMac, long tunnelOFPort, long localPort) {
 332         /*
 333          * Table(0) Rule #2
 334          * ----------------
 335          * Match: Ingress Port, Tunnel ID
 336          * Action: GOTO Local Table (20)
 337          */
 338
 339         handleTunnelIn(dpid, TABLE_0_DEFAULT_INGRESS, TABLE_2_LOCAL_FORWARD, segmentationId, tunnelOFPort, true);
 340
 341         /*
 342          * Table(1) Rule #2
 343          * ----------------
 344          * Match: Match Tunnel ID and L2 ::::FF:FF Flooding
 345          * Action: Flood to selected destination TEPs
 346          * -------------------------------------------
 347          * table=1,priority=16384,tun_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
 348          * actions=output:10,output:11,goto_table:2
 349          */
 350
 351         handleTunnelFloodOut(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD, segmentationId, tunnelOFPort, true);
 352
 353     }
 354
 355     private void programRemoteEgressTunnelBridgeRules(Node node, Long dpid, String segmentationId, String attachedMac, long tunnelOFPort, long localPort) {
 356         /*
 357          * Table(1) Rule #1
 358          * ----------------
 359          * Match: Drop any remaining Ingress Local VM Packets
 360          * Action: Drop w/ a low priority
 361          * -------------------------------------------
 362          * table=1,tun_id=0x5,dl_dst=00:00:00:00:00:08 \
 363          * actions=output:11,goto_table:2
 364          */
 365
 366         handleTunnelOut(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD, segmentationId, tunnelOFPort, attachedMac, true);
 367     }
 368
 369     private void removeRemoteEgressTunnelBridgeRules(Node node, Long dpid, String segmentationId, String attachedMac, long tunnelOFPort, long localPort) {
 370         /*
 371          * Table(1) Rule #1
 372          * ----------------
 373          * Match: Drop any remaining Ingress Local VM Packets
 374          * Action: Drop w/ a low priority
 375          * -------------------------------------------
 376          * table=1,tun_id=0x5,dl_dst=00:00:00:00:00:08 \
 377          * actions=output:11,goto_table:2
 378          */
 379
 380         handleTunnelOut(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD, segmentationId, tunnelOFPort, attachedMac, false);
 381     }
 382
 383     /* Remove tunnel rules if last node in this tenant network */
 384     private void removePerTunnelRules(Node node, Long dpid, String segmentationId, long tunnelOFPort) {
 385         /*
 386          * TODO : Optimize the following 2 writes to be restricted only for the very first port known in a segment.
 387          */
 388         /*
 389          * Table(1) Rule #3
 390          * ----------------
 391          * Match:  Any remaining Ingress Local VM Packets
 392          * Action: Drop w/ a low priority
 393          * -------------------------------------------
 394          * table=1,priority=8192,tun_id=0x5 actions=goto_table:2
 395          */
 396
 397         handleTunnelMiss(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD, segmentationId, false);
 398
 399         /*
 400          * Table(2) Rule #3
 401          * ----------------
 402          * Match: Any Remaining Flows w/a TunID
 403          * Action: Drop w/ a low priority
 404          * table=2,priority=8192,tun_id=0x5 actions=drop
 405          */
 406
 407         handleLocalTableMiss(dpid, TABLE_2_LOCAL_FORWARD, segmentationId, false);
 408
 409         /*
 410          * Table(0) Rule #2
 411          * ----------------
 412          * Match: Ingress Port, Tunnel ID
 413          * Action: GOTO Local Table (10)
 414          */
 415
 416         handleTunnelIn(dpid, TABLE_0_DEFAULT_INGRESS, TABLE_2_LOCAL_FORWARD, segmentationId, tunnelOFPort, false);
 417
 418         /*
 419          * Table(1) Rule #2
 420          * ----------------
 421          * Match: Match Tunnel ID and L2 ::::FF:FF Flooding
 422          * Action: Flood to selected destination TEPs
 423          * -------------------------------------------
 424          * table=1,priority=16384,tun_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
 425          * actions=output:10,output:11,goto_table:2
 426          */
 427
 428         handleTunnelFloodOut(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD, segmentationId, tunnelOFPort, false);
 429     }
 430
 431     private void programLocalVlanRules(Node node, Long dpid, String segmentationId, String attachedMac, long localPort) {
 432         /*
 433          * Table(0) Rule #1
 434          * ----------------
 435          * Tag traffic coming from the local port and vm srcmac
 436          * Match: VM sMac and Local Ingress Port
 437          * Action: Set VLAN ID and GOTO Local Table 1
 438          */
 439
 440         handleLocalInPortSetVlan(dpid, TABLE_0_DEFAULT_INGRESS,
 441                 TABLE_1_ISOLATE_TENANT, segmentationId, localPort,
 442                 attachedMac, true);
 443
 444         /*
 445          * Table(0) Rule #3
 446          * ----------------
 447          * Drop all other traffic coming from the local port
 448          * Match: Drop any remaining Ingress Local VM Packets
 449          * Action: Drop w/ a low priority
 450          */
 451
 452         handleDropSrcIface(dpid, localPort, true);
 453
 454         /*
 455          * Table(2) Rule #1
 456          * ----------------
 457          * Forward unicast traffic destined to the local port after stripping tag
 458          * Match: Match VLAN ID and Destination DL/dMAC Addr
 459          * Action: strip vlan, output to local port
 460          * Example: table=2,vlan_id=0x5,dl_dst=00:00:00:00:00:01 actions= strip vlan, output:2
 461          */
 462
 463         handleLocalVlanUcastOut(dpid, TABLE_2_LOCAL_FORWARD, segmentationId,
 464                 localPort, attachedMac, true);
 465
 466         /*
 467          * Table(2) Rule #2
 468          * ----------------
 469          * Match: VLAN ID and dMAC (::::FF:FF)
 470          * Action: strip vlan, output to all local ports in this vlan
 471          * Example: table=2,priority=16384,vlan_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
 472          * actions= strip_vlan, output:2,3,4,5
 473          */
 474
 475         //handleLocalVlanBcastOut(dpid, TABLE_2_LOCAL_FORWARD, segmentationId,
 476         //        localPort, ethPort, true);
 477         //handleVlanFloodOut(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD,
 478         //        segmentationId, localPort, ethport, true);
 479
 480         /*
 481          * Table(2) Rule #3
 482          * ----------------
 483          * Match: Any Remaining Flows w/a VLAN ID
 484          * Action: Drop w/ a low priority
 485          * Example: table=2,priority=8192,vlan_id=0x5 actions=drop
 486          */
 487
 488         //handleLocalVlanTableMiss(dpid, TABLE_2_LOCAL_FORWARD, segmentationId,
 489         //        true);
 490     }
 491
 492     private void removeLocalVlanRules(Node node, Long dpid,
 493                                       String segmentationId, String attachedMac, long localPort) {
 494         /*
 495          * Table(0) Rule #1
 496          * ----------------
 497          * Match: VM sMac and Local Ingress Port
 498          * Action: Set VLAN ID and GOTO Local Table 1
 499          */
 500
 501         handleLocalInPortSetVlan(dpid, TABLE_0_DEFAULT_INGRESS,
 502                 TABLE_1_ISOLATE_TENANT, segmentationId, localPort,
 503                 attachedMac, false);
 504
 505         /*
 506          * Table(0) Rule #3
 507          * ----------------
 508          * Match: Drop any remaining Ingress Local VM Packets
 509          * Action: Drop w/ a low priority
 510          */
 511
 512         handleDropSrcIface(dpid, localPort, false);
 513
 514         /*
 515          * Table(2) Rule #1
 516          * ----------------
 517          * Match: Match VLAN ID and Destination DL/dMAC Addr
 518          * Action: strip vlan, output to local port
 519          * Example: table=2,vlan_id=0x5,dl_dst=00:00:00:00:00:01 actions= strip vlan, output:2
 520          */
 521
 522         handleLocalVlanUcastOut(dpid, TABLE_2_LOCAL_FORWARD, segmentationId,
 523                 localPort, attachedMac, false);
 524
 525         /*
 526          * Table(2) Rule #2
 527          * ----------------
 528          * Match: VLAN ID and dMAC (::::FF:FF)
 529          * Action: strip vlan, output to all local ports in this vlan
 530          * Example: table=2,priority=16384,vlan_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
 531          * actions= strip_vlan, output:2,3,4,5
 532          */
 533
 534         //handleLocalVlanBcastOut(dpid, TABLE_2_LOCAL_FORWARD, segmentationId,
 535         //        localPort, ethPort, false);
 536         //handleVlanFloodOut(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD,
 537         //        segmentationId, localPort, false);
 538
 539     }
 540
 541     private void programLocalIngressVlanRules(Node node, Long dpid, String segmentationId, String attachedMac,
 542                                               long localPort, long ethPort) {
 543         /*
 544          * Table(0) Rule #2
 545          * ----------------
 546          * Match: Ingress port = physical interface, Vlan ID
 547          * Action: GOTO Local Table 2
 548          */
 549
 550         handleVlanIn(dpid, TABLE_0_DEFAULT_INGRESS, TABLE_2_LOCAL_FORWARD,
 551                 segmentationId, ethPort, true);
 552
 553         /*
 554          * Table(1) Rule #2
 555          * ----------------
 556          * Match: Match VLAN ID and L2 ::::FF:FF Flooding
 557          * Action: Flood to local and remote VLAN members
 558          * -------------------------------------------
 559          * Example: table=1,priority=16384,vlan_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
 560          * actions=output:10 (eth port),goto_table:2
 561          * table=110, priority=16384,dl_vlan=2001,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=output:2,pop_vlan,output:1,output:3,output:4
 562          */
 563
 564         handleLocalVlanBcastOut(dpid, TABLE_2_LOCAL_FORWARD, segmentationId, localPort, ethPort, true);
 565
 566         /*
 567          * Table(1) Rule #2
 568          * ----------------
 569          * Match: Match VLAN ID and L2 ::::FF:FF Flooding
 570          * Action: Flood to local and remote VLAN members
 571          * -------------------------------------------
 572          * Example: table=1,priority=16384,vlan_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
 573          * actions=output:10 (eth port),goto_table:2
 574          */
 575
 576         //handleVlanFloodOut(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD,
 577         //        segmentationId, ethPort, true);
 578     }
 579
 580     private void programRemoteEgressVlanRules(Node node, Long dpid, String segmentationId,
 581                                               String attachedMac, long ethPort) {
 582         /*
 583          * Table(1) Rule #1
 584          * ----------------
 585          * Match: Destination MAC is local VM MAC and vlan id
 586          * Action: go to table 2
 587          * -------------------------------------------
 588          * Example: table=1,vlan_id=0x5,dl_dst=00:00:00:00:00:08 \
 589          * actions=goto_table:2
 590          */
 591
 592         //handleVlanOut(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD,
 593         //        segmentationId, ethPort, attachedMac, true);
 594
 595         /*
 596          * Table(1) Rule #3
 597          * ----------------
 598          * Match:  VLAN ID
 599          * Action: Go to table 2
 600          * -------------------------------------------
 601          * Example: table=1,priority=8192,vlan_id=0x5 actions=output:1,goto_table:2
 602          * table=110,priority=8192,dl_vlan=2001 actions=output:2
 603          */
 604
 605         handleVlanMiss(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD, segmentationId, ethPort, true);
 606     }
 607
 608     private void removeRemoteEgressVlanRules(Node node, Long dpid, String segmentationId,
 609                                              String attachedMac, long localPort, long ethPort) {
 610         /*
 611          * Table(1) Rule #1
 612          * ----------------
 613          * Match: Destination MAC is local VM MAC and vlan id
 614          * Action: go to table 2
 615          * -------------------------------------------
 616          * Example: table=1,vlan_id=0x5,dl_dst=00:00:00:00:00:08 \
 617          * actions=goto_table:2
 618          */
 619
 620         //handleVlanOut(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD,
 621         //        segmentationId, ethPort, attachedMac, false);
 622
 623         /*
 624          * Table(1) Rule #2
 625          * ----------------
 626          * Match: Match VLAN ID and L2 ::::FF:FF Flooding
 627          * Action: Flood to local and remote VLAN members
 628          * -------------------------------------------
 629          * Example: table=1,priority=16384,vlan_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
 630          * actions=output:10 (eth port),goto_table:2
 631          * table=110, priority=16384,dl_vlan=2001,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=output:2,pop_vlan,output:1,output:3,output:4
 632          */
 633
 634         handleLocalVlanBcastOut(dpid, TABLE_2_LOCAL_FORWARD, segmentationId, localPort, ethPort, false);
 635     }
 636
 637     private void removePerVlanRules(Node node, Long dpid, String segmentationId, long localPort, long ethPort) {
 638         /*
 639          * Table(2) Rule #3
 640          * ----------------
 641          * Match: Any Remaining Flows w/a VLAN ID
 642          * Action: Drop w/ a low priority
 643          * Example: table=2,priority=8192,vlan_id=0x5 actions=drop
 644          */
 645
 646         //handleLocalVlanTableMiss(dpid, TABLE_2_LOCAL_FORWARD, segmentationId, false);
 647
 648         /*
 649          * Table(0) Rule #2
 650          * ----------------
 651          * Match: Ingress port = physical interface, Vlan ID
 652          * Action: GOTO Local Table 2
 653          */
 654
 655         handleVlanIn(dpid, TABLE_0_DEFAULT_INGRESS, TABLE_2_LOCAL_FORWARD, segmentationId, ethPort, false);
 656
 657         /*
 658          * Table(1) Rule #2
 659          * ----------------
 660          * Match: Match VLAN ID and L2 ::::FF:FF Flooding
 661          * Action: Flood to local and remote VLAN members
 662          * -------------------------------------------
 663          * Example: table=1,priority=16384,vlan_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
 664          * actions=output:10 (eth port),goto_table:2
 665          * table=110, priority=16384,dl_vlan=2001,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=output:2,pop_vlan,output:1,output:3,output:4
 666          */
 667
 668         //handleLocalVlanBcastOut(dpid, TABLE_2_LOCAL_FORWARD, segmentationId, localPort, ethPort, false);
 669
 670         /*
 671          * Table(1) Rule #2
 672          * ----------------
 673          * Match: Match VLAN ID and L2 ::::FF:FF Flooding
 674          * Action: Flood to local and remote VLAN members
 675          * -------------------------------------------
 676          * Example: table=1,priority=16384,vlan_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
 677          * actions=output:10 (eth port),goto_table:2
 678          */
 679
 680         //handleVlanFloodOut(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD,
 681         //        segmentationId, ethPort, false);
 682
 683         /*
 684          * Table(1) Rule #3
 685          * ----------------
 686          * Match:  VLAN ID
 687          * Action: Go to table 2
 688          * -------------------------------------------
 689          * Example: table=1,priority=8192,vlan_id=0x5 actions=output:1,goto_table:2
 690          * table=110,priority=8192,dl_vlan=2001 actions=output:2
 691          */
 692
 693         handleVlanMiss(dpid, TABLE_1_ISOLATE_TENANT, TABLE_2_LOCAL_FORWARD, segmentationId, ethPort, false);
 694     }
 695
 696     private long getDpid(Node node) {
 697         long dpid = southbound.getDataPathId(node);
 698         if (dpid == 0) {
 699             LOG.warn("getDpid: dpid not found: {}", node);
 700         }
 701         return dpid;
 702     }
 703
 704     private long getIntegrationBridgeOFDPID(Node node) {
 705         long dpid = 0L;
 706         if (southbound.getBridgeName(node).equals(configurationService.getIntegrationBridgeName())) {
 707             dpid = getDpid(node);
 708         }
 709         return dpid;
 710     }
 711
 712     /**
 713      * Returns true is the network if of type GRE or VXLAN
 714      *
 715      * @param networkType The type of the network
 716      * @return returns true if the network is a tunnel
 717      */
 718     private boolean isTunnel(String networkType)
 719     {
 720         return (networkType.equalsIgnoreCase(NetworkHandler.NETWORK_TYPE_GRE) || networkType.equalsIgnoreCase
 721                 (NetworkHandler.NETWORK_TYPE_VXLAN));
 722     }
 723
 724     /**
 725      * Returns true if the network is of type vlan.
 726      *
 727      * @param networkType The type of the network
 728      * @return returns true if the network is a vlan
 729      */
 730     private boolean isVlan(String networkType)
 731     {
 732         return networkType.equalsIgnoreCase(NetworkHandler.NETWORK_TYPE_VLAN);
 733     }
 734
 735     private void programLocalRules(String networkType, String segmentationId, Node node,
 736                                     OvsdbTerminationPointAugmentation intf) {
 737         LOG.debug("programLocalRules: node: {}, intf: {}, networkType: {}, segmentationId: {}",
 738                 node.getNodeId(), intf.getName(), networkType, segmentationId);
 739         try {
 740             long dpid = getIntegrationBridgeOFDPID(node);
 741             if (dpid == 0L) {
 742                 LOG.debug("programLocalRules: Openflow Datapath-ID not set for the integration bridge in {}",
 743                         node);
 744                 return;
 745             }
 746
 747             long localPort = southbound.getOFPort(intf);
 748             if (localPort == 0) {
 749                 LOG.info("programLocalRules: could not find ofPort for Port {} on Node {}",
 750                          intf.getName(), node.getNodeId());
 751                 return;
 752             }
 753
 754             String attachedMac = southbound.getInterfaceExternalIdsValue(intf, Constants.EXTERNAL_ID_VM_MAC);
 755             if (attachedMac == null) {
 756                 LOG.warn("No AttachedMac seen in {}", intf);
 757                 return;
 758             }
 759
 760             /* Program local rules based on network type */
 761             if (isVlan(networkType)) {
 762                 LOG.debug("Program local vlan rules for interface {}", intf.getName());
 763                 programLocalVlanRules(node, dpid, segmentationId, attachedMac, localPort);
 764             }
 765             if ((isTunnel(networkType) || isVlan(networkType))) {
 766                 programLocalSecurityGroupRules(attachedMac, node, intf, dpid, localPort, segmentationId, true);
 767             }
 768             if (isTunnel(networkType)) {
 769                 LOG.debug("Program local bridge rules for interface {}, "
 770                                 + "dpid: {}, segmentationId: {}, attachedMac: {}, localPort: {}",
 771                         intf.getName(), dpid, segmentationId, attachedMac, localPort);
 772                 programLocalBridgeRules(node, dpid, segmentationId, attachedMac, localPort);
 773             }
 774         } catch (Exception e) {
 775             LOG.error("Exception in programming Local Rules for {} on {}", intf, node, e);
 776         }
 777     }
 778
 779     private void removeLocalRules(String networkType, String segmentationId, Node node,
 780                                    OvsdbTerminationPointAugmentation intf) {
 781         LOG.debug("removeLocalRules: node: {}, intf: {}, networkType: {}, segmentationId: {}",
 782                 node.getNodeId(), intf.getName(), networkType, segmentationId);
 783         try {
 784             long dpid = getIntegrationBridgeOFDPID(node);
 785             if (dpid == 0L) {
 786                 LOG.debug("removeLocalRules: Openflow Datapath-ID not set for the integration bridge in {}", node);
 787                 return;
 788             }
 789
 790             long localPort = southbound.getOFPort(intf);
 791             if (localPort == 0) {
 792                 LOG.info("removeLocalRules: could not find ofPort");
 793                 return;
 794             }
 795
 796             String attachedMac = southbound.getInterfaceExternalIdsValue(intf, Constants.EXTERNAL_ID_VM_MAC);
 797             if (attachedMac == null) {
 798                 LOG.warn("No AttachedMac seen in {}", intf);
 799                 return;
 800             }
 801
 802             /* Program local rules based on network type */
 803             if (isVlan(networkType)) {
 804                 LOG.debug("Remove local vlan rules for interface {}", intf.getName());
 805                 removeLocalVlanRules(node, dpid, segmentationId, attachedMac, localPort);
 806             } else if (isTunnel(networkType)) {
 807                 LOG.debug("Remove local bridge rules for interface {}", intf.getName());
 808                 removeLocalBridgeRules(node, dpid, segmentationId, attachedMac, localPort);
 809             }
 810             if (isTunnel(networkType) || isVlan(networkType)) {
 811                 programLocalSecurityGroupRules(attachedMac, node, intf, dpid, localPort, segmentationId, false);
 812             }
 813         } catch (Exception e) {
 814             LOG.error("Exception in removing Local Rules for {} on {}", intf, node, e);
 815         }
 816     }
 817
 818     // TODO SB_MIGRATION
 819     // Need to handle case where a node comes online after a network and tunnels have
 820     // already been created. The interface update is what triggers creating the l2 forwarding flows
 821     // so we don't see those updates in this case - we only see the new nodes interface updates.
 822     private void programTunnelRules (String tunnelType, String segmentationId, InetAddress dst, Node node,
 823                                      OvsdbTerminationPointAugmentation intf, boolean local) {
 824         LOG.debug("programTunnelRules: node: {}, intf: {}, local: {}, tunnelType: {}, "
 825                         + "segmentationId: {}, dstAddr: {}",
 826                 node.getNodeId(), intf.getName(), local, tunnelType, segmentationId, dst.getHostAddress());
 827         try {
 828             long dpid = getIntegrationBridgeOFDPID(node);
 829             if (dpid == 0L) {
 830                 LOG.debug("programTunnelRules: Openflow Datapath-ID not set for the integration bridge in {}", node);
 831                 return;
 832             }
 833
 834             long localPort = southbound.getOFPort(intf);
 835             if (localPort == 0) {
 836                 LOG.info("programTunnelRules: could not find ofPort for Port {} on Node{}", intf.getName(), node.getNodeId());
 837                 return;
 838             }
 839
 840             String attachedMac = southbound.getInterfaceExternalIdsValue(intf, Constants.EXTERNAL_ID_VM_MAC);
 841             if (attachedMac == null) {
 842                 LOG.warn("programTunnelRules: No AttachedMac seen in {}", intf);
 843                 return;
 844             }
 845
 846             OvsdbTerminationPointAugmentation tunnelPort= southbound.getTerminationPointOfBridge(node, getTunnelName(tunnelType, dst));
 847             if(tunnelPort != null){
 848                 long tunnelOFPort = southbound.getOFPort(tunnelPort);
 849                 if (tunnelOFPort == 0) {
 850                     LOG.error("programTunnelRules: Could not Identify Tunnel port {} -> OF ({}) on {}",
 851                             tunnelPort.getName(), tunnelOFPort, node);
 852                     return;
 853                 }
 854                 LOG.debug("programTunnelRules: Identified Tunnel port {} -> OF ({}) on {}",
 855                         tunnelPort.getName(), tunnelOFPort, node);
 856
 857                 if (!local) {
 858                     LOG.trace("programTunnelRules: program remote egress tunnel rules: node {}, intf {}",
 859                             node.getNodeId().getValue(), intf.getName());
 860                     programRemoteEgressTunnelBridgeRules(node, dpid, segmentationId, attachedMac,
 861                             tunnelOFPort, localPort);
 862                 } else {
 863                     LOG.trace("programTunnelRules: program local ingress tunnel rules: node {}, intf {}",
 864                             node.getNodeId().getValue(), intf.getName());
 865                     programLocalIngressTunnelBridgeRules(node, dpid, segmentationId, attachedMac,
 866                             tunnelOFPort, localPort);
 867                 }
 868             }
 869         } catch (Exception e) {
 870             LOG.warn("Failed to program tunnel rules, node {}, intf {}", node, intf, e);
 871         }
 872     }
 873
 874     private void removeTunnelRules (String tunnelType, String segmentationId, InetAddress dst, Node node,
 875                                     OvsdbTerminationPointAugmentation intf,
 876                                     boolean local, boolean isLastInstanceOnNode) {
 877         LOG.debug("removeTunnelRules: node: {}, intf: {}, local: {}, tunnelType: {}, "
 878                         + "segmentationId: {}, dstAddr: {}, isLastinstanceOnNode: {}",
 879                 node.getNodeId(), intf.getName(), local, tunnelType, segmentationId, dst, isLastInstanceOnNode);
 880         try {
 881             long dpid = getIntegrationBridgeOFDPID(node);
 882             if (dpid == 0L) {
 883                 LOG.debug("removeTunnelRules: Openflow Datapath-ID not set for the integration bridge in {}", node);
 884                 return;
 885             }
 886
 887             long localPort = southbound.getOFPort(intf);
 888             if (localPort == 0) {
 889                 LOG.info("removeTunnelRules: could not find ofPort");
 890                 return;
 891             }
 892
 893             String attachedMac = southbound.getInterfaceExternalIdsValue(intf, Constants.EXTERNAL_ID_VM_MAC);
 894             if (attachedMac == null) {
 895                 LOG.error("removeTunnelRules: No AttachedMac seen in {}", intf);
 896                 return;
 897             }
 898
 899             List<OvsdbTerminationPointAugmentation> intfs = southbound.getTerminationPointsOfBridge(node);
 900             for (OvsdbTerminationPointAugmentation tunIntf : intfs) {
 901                 if (tunIntf.getName().equals(getTunnelName(tunnelType, dst))) {
 902                     long tunnelOFPort = southbound.getOFPort(tunIntf);
 903                     if (tunnelOFPort == 0) {
 904                         LOG.error("Could not Identify Tunnel port {} -> OF ({}) on {}",
 905                                 tunIntf.getName(), tunnelOFPort, node);
 906                         return;
 907                     }
 908                     LOG.debug("Identified Tunnel port {} -> OF ({}) on {}",
 909                             tunIntf.getName(), tunnelOFPort, node);
 910
 911                     if (!local) {
 912                         removeRemoteEgressTunnelBridgeRules(node, dpid, segmentationId, attachedMac,
 913                                 tunnelOFPort, localPort);
 914                     }
 915                     if (local && isLastInstanceOnNode) {
 916                         removePerTunnelRules(node, dpid, segmentationId, tunnelOFPort);
 917                     }
 918                     return;
 919                 }
 920             }
 921         } catch (Exception e) {
 922             LOG.error("Failed to remove tunnel rules, node {}, intf {}", node, intf, e);
 923         }
 924     }
 925
 926     private void programVlanRules (NeutronNetwork network, Node node, OvsdbTerminationPointAugmentation intf) {
 927         LOG.debug("programVlanRules: node: {}, network: {}, intf: {}",
 928                 node.getNodeId(), network.getNetworkUUID(), intf.getName());
 929         long dpid = getIntegrationBridgeOFDPID(node);
 930         if (dpid == 0L) {
 931             LOG.debug("programVlanRules: Openflow Datapath-ID not set for the integration bridge in {}", node);
 932             return;
 933         }
 934
 935         long localPort = southbound.getOFPort(intf);
 936         if (localPort == 0) {
 937             LOG.debug("programVlanRules: could not find ofPort for {}", intf.getName());
 938             return;
 939         }
 940
 941         String attachedMac = southbound.getInterfaceExternalIdsValue(intf, Constants.EXTERNAL_ID_VM_MAC);
 942         if (attachedMac == null) {
 943             LOG.debug("programVlanRules: No AttachedMac seen in {}", intf);
 944             return;
 945         }
 946
 947         String phyIfName =
 948                 bridgeConfigurationManager.getPhysicalInterfaceName(node, network.getProviderPhysicalNetwork());
 949         long ethOFPort = southbound.getOFPort(node, phyIfName);
 950         if (ethOFPort == 0) {
 951             LOG.warn("programVlanRules: could not find ofPort for physical port {}", phyIfName);
 952             return;
 953         }
 954         LOG.debug("programVlanRules: Identified eth port {} -> ofPort ({}) on {}",
 955                 phyIfName, ethOFPort, node);
 956         // TODO: add logic to only add rule on remote nodes
 957         programRemoteEgressVlanRules(node, dpid, network.getProviderSegmentationID(),
 958                 attachedMac, ethOFPort);
 959         programLocalIngressVlanRules(node, dpid, network.getProviderSegmentationID(),
 960                 attachedMac, localPort, ethOFPort);
 961     }
 962
 963     private void removeVlanRules (NeutronNetwork network, Node node, OvsdbTerminationPointAugmentation intf,
 964                                   boolean isLastInstanceOnNode) {
 965         LOG.debug("removeVlanRules: node: {}, network: {}, intf: {}, isLastInstanceOnNode",
 966                 node.getNodeId(), network.getNetworkUUID(), intf.getName(), isLastInstanceOnNode);
 967         long dpid = getIntegrationBridgeOFDPID(node);
 968         if (dpid == 0L) {
 969             LOG.debug("removeVlanRules: Openflow Datapath-ID not set for the integration bridge in {}", node);
 970             return;
 971         }
 972
 973         long localPort = southbound.getOFPort(intf);
 974         if (localPort == 0) {
 975             LOG.debug("removeVlanRules: programVlanRules: could not find ofPort for {}", intf.getName());
 976             return;
 977         }
 978
 979         String attachedMac = southbound.getInterfaceExternalIdsValue(intf, Constants.EXTERNAL_ID_VM_MAC);
 980         if (attachedMac == null) {
 981             LOG.debug("removeVlanRules: No AttachedMac seen in {}", intf);
 982             return;
 983         }
 984
 985         String phyIfName =
 986                 bridgeConfigurationManager.getPhysicalInterfaceName(node, network.getProviderPhysicalNetwork());
 987         long ethOFPort = southbound.getOFPort(node, phyIfName);
 988         if (ethOFPort == 0) {
 989             LOG.warn("removeVlanRules: could not find ofPort for physical port {}", phyIfName);
 990             return;
 991         }
 992         LOG.debug("removeVlanRules: Identified eth port {} -> ofPort ({}) on {}",
 993                 phyIfName, ethOFPort, node);
 994
 995         removeRemoteEgressVlanRules(node, dpid, network.getProviderSegmentationID(),
 996                 attachedMac, localPort, ethOFPort);
 997         if (isLastInstanceOnNode) {
 998             removePerVlanRules(node, dpid, network.getProviderSegmentationID(), localPort, ethOFPort);
 999         }
1000     }
1001
1002     private void programLocalSecurityGroupRules(String attachedMac, Node node, OvsdbTerminationPointAugmentation intf,
1003                                  Long dpid,long localPort, String segmentationId,
1004                                  boolean write) {
1005
1006         LOG.debug("programLocalRules: Program fixed security group rules for interface {}", intf.getName());
1007         NeutronPort dhcpPort = securityServicesManager.getDhcpServerPort(intf);
1008         boolean isComputePort = false;
1009         boolean isLastPortinBridge = false;
1010         boolean isLastPortinSubnet = false;
1011         List<Neutron_IPs> srcAddressList = null;
1012         if (null != dhcpPort) {
1013             isComputePort = securityServicesManager.isComputePort(intf);
1014             isLastPortinBridge = securityServicesManager.isLastPortinBridge(node, intf);
1015             isLastPortinSubnet = false;
1016             if (isComputePort) {
1017                 isLastPortinSubnet = securityServicesManager.isLastPortinSubnet(node, intf);
1018                 srcAddressList = securityServicesManager.getIpAddressList(intf);
1019                 if (null == srcAddressList) {
1020                     LOG.warn("programLocalRules: No Ip address assigned {}", intf);
1021                     return;
1022                 }
1023             }
1024             ingressAclProvider.programFixedSecurityGroup(dpid, segmentationId, dhcpPort.getMacAddress(), localPort,
1025                                                        isLastPortinSubnet, isComputePort, attachedMac, write);
1026             egressAclProvider.programFixedSecurityGroup(dpid, segmentationId, attachedMac, localPort,
1027                                                       srcAddressList, isLastPortinBridge, isComputePort,write);
1028             /* If the network type is tunnel based (VXLAN/GRRE/etc) with Neutron Port Security ACLs */
1029             /* TODO SB_MIGRATION */
1030
1031             LOG.debug("Neutron port has a Port Security Group");
1032             // Retrieve the security group from the Neutron Port and apply the rules
1033             if (securityServicesManager.isPortSecurityReady(intf)) {
1034                 //Associate the security group flows.
1035                 List<NeutronSecurityGroup> securityGroupListInPort = securityServicesManager
1036                         .getSecurityGroupInPortList(intf);
1037                 String neutronPortId = southbound.getInterfaceExternalIdsValue(intf,
1038                                                                                Constants.EXTERNAL_ID_INTERFACE_ID);
1039                 for (NeutronSecurityGroup securityGroupInPort:securityGroupListInPort) {
1040                     ingressAclProvider.programPortSecurityGroup(dpid, segmentationId, attachedMac, localPort,
1041                                                               securityGroupInPort, neutronPortId, write);
1042                     egressAclProvider.programPortSecurityGroup(dpid, segmentationId, attachedMac, localPort,
1043                                                              securityGroupInPort, neutronPortId, write);
1044                 }
1045             }
1046         } else {
1047             LOG.warn("programLocalRules: No DCHP port seen in  network of {}", intf);
1048         }
1049     }
1050
1051     /*
1052      * The function is for the new compute node joining the existing network.
1053      * When a new VM is instantiated in the new compute node, neutron port add
1054      * event is generated. This event is processed only for that node. So,
1055      * loop through all the ports of the same network and install unicast mac
1056      * flow for the VM's created on the TEP of the destination node in src node.
1057      * This function will be executed even for any new VM creation in an existing
1058      * network. If a cache is maintained to optimize the below flow addition, it will
1059      * work only for one unstack and restack. For the next unstack and restack,
1060      * it will not work since the cache would have been already deleted.
1061      */
1062     private void programTunnelRulesInNewNode(NeutronNetwork network,
1063                                              String networkType, String segmentationId,
1064                                              InetAddress src, InetAddress dst,
1065                                              Node srcBridgeNode, Node dstBridgeNode,
1066                                              OvsdbTerminationPointAugmentation intf){
1067         try {
1068             long localPort = southbound.getOFPort(intf);
1069             if(localPort != 0)
1070             {
1071                 LOG.debug("Interface update details {}", intf);
1072
1073                 /*
1074                  * When a network is added and the TEP destination is not present in a
1075                  * node C1, tunnelin and broadcast rules will not be programmed, since
1076                  * OF port is not created. So, when a new node C2 joins and create a new
1077                  * VM, the tunnelin and broadcast rule will not be present in C1.
1078                  * So, handling it in the case below to make ping work.
1079                  */
1080                 if(securityServicesManager.getNeutronPortFromDhcpIntf(intf) == null){
1081                     programTunnelRules(networkType, segmentationId, src, dstBridgeNode, intf, true);
1082                 }
1083
1084                 /*
1085                  * FIX for 4208 - loop through all the ports and add the VM's
1086                  * unicast mac rule of the destination node in the source node.
1087                  * When a new node is added, it needs to configure the VM unicast mac
1088                  * flow rules which were created before it was joined to an existing
1089                  * network.
1090                  */
1091                 List<OvsdbTerminationPointAugmentation> ports = southbound.getTerminationPointsOfBridge(dstBridgeNode);
1092                 for (OvsdbTerminationPointAugmentation port : ports) {
1093                     if(network == tenantNetworkManager.getTenantNetwork(port)){
1094                         programTunnelRules(networkType, segmentationId, dst, srcBridgeNode, port, false);
1095                     }
1096                     else{
1097                         LOG.trace("Port {} is not part of network {}", port, network);
1098                     }
1099                 }
1100             }
1101         } catch (Exception e) {
1102             LOG.error("Exception during handlingNeutron network add", e);
1103         }
1104     }
1105
1106     @Override
1107     public boolean handleInterfaceUpdate(NeutronNetwork network, Node srcNode,
1108                                          OvsdbTerminationPointAugmentation intf) {
1109         Preconditions.checkNotNull(nodeCacheManager);
1110         Map<org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId,Node> nodes =
1111                 nodeCacheManager.getOvsdbNodes();
1112         nodes.remove(southbound.extractBridgeOvsdbNodeId(srcNode));
1113         String networkType = network.getProviderNetworkType();
1114         String segmentationId = network.getProviderSegmentationID();
1115         Node srcBridgeNode = southbound.getBridgeNode(srcNode, configurationService.getIntegrationBridgeName());
1116         programLocalRules(networkType, network.getProviderSegmentationID(), srcBridgeNode, intf);
1117
1118         if (isVlan(networkType)) {
1119             programVlanRules(network, srcNode, intf);
1120         } else if (isTunnel(networkType)){
1121
1122             boolean sourceTunnelStatus = false;
1123             boolean destTunnelStatus = false;
1124             for (Node dstNode : nodes.values()) {
1125                 InetAddress src = configurationService.getTunnelEndPoint(srcNode);
1126                 InetAddress dst = configurationService.getTunnelEndPoint(dstNode);
1127                 if ((src != null) && (dst != null)) {
1128                     sourceTunnelStatus = addTunnelPort(srcBridgeNode, networkType, src, dst);
1129
1130                     Node dstBridgeNode = southbound.getBridgeNode(dstNode,
1131                             configurationService.getIntegrationBridgeName());
1132
1133                     if(dstBridgeNode != null){
1134                         destTunnelStatus = addTunnelPort(dstBridgeNode, networkType, dst, src);
1135                     }
1136
1137                     if (sourceTunnelStatus) {
1138                         programTunnelRules(networkType, segmentationId, dst, srcBridgeNode, intf, true);
1139                     }
1140                     if (destTunnelStatus) {
1141                         programTunnelRules(networkType, segmentationId, src, dstBridgeNode, intf, false);
1142                         programTunnelRulesInNewNode(network, networkType, segmentationId, src, dst,
1143                                                     srcBridgeNode, dstBridgeNode, intf);
1144                     }
1145                 } else {
1146                     LOG.warn("Tunnel end-point configuration missing. Please configure it in OpenVSwitch Table. "
1147                                     + "Check source {} or destination {}",
1148                             src != null ? src.getHostAddress() : "null",
1149                             dst != null ? dst.getHostAddress() : "null");
1150                 }
1151             }
1152         }
1153
1154         return true;
1155     }
1156
1157     private void triggerInterfaceUpdates(Node node) {
1158         LOG.debug("enter triggerInterfaceUpdates for {}", node.getNodeId());
1159         List<OvsdbTerminationPointAugmentation> ports = southbound.extractTerminationPointAugmentations(node);
1160         if (ports != null && !ports.isEmpty()) {
1161             for (OvsdbTerminationPointAugmentation port : ports) {
1162                 NeutronNetwork neutronNetwork = tenantNetworkManager.getTenantNetwork(port);
1163                 if (neutronNetwork != null) {
1164                     LOG.warn("Trigger Interface update for {}", port);
1165                     handleInterfaceUpdate(neutronNetwork, node, port);
1166                 }
1167             }
1168         } else {
1169             LOG.warn("triggerInterfaceUpdates: tps are null");
1170         }
1171         LOG.debug("exit triggerInterfaceUpdates for {}", node.getNodeId());
1172     }
1173
1174     @Override
1175     public boolean handleInterfaceDelete(String tunnelType, NeutronNetwork network, Node srcNode,
1176                                          OvsdbTerminationPointAugmentation intf, boolean isLastInstanceOnNode) {
1177         Map<org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId,Node> nodes =
1178                 nodeCacheManager.getOvsdbNodes();
1179         nodes.remove(southbound.extractBridgeOvsdbNodeId(srcNode));
1180
1181         LOG.info("Delete intf " + intf.getName() + " isLastInstanceOnNode " + isLastInstanceOnNode);
1182         List<String> phyIfName = bridgeConfigurationManager.getAllPhysicalInterfaceNames(srcNode);
1183         if (southbound.isTunnel(intf)) {
1184             // Delete tunnel port
1185             try {
1186                 InetAddress src = InetAddress.getByName(
1187                         southbound.getOptionsValue(intf.getOptions(), "local_ip"));
1188                 InetAddress dst = InetAddress.getByName(
1189                         southbound.getOptionsValue(intf.getOptions(), "remote_ip"));
1190                 deleteTunnelPort(srcNode,
1191                         MdsalHelper.createOvsdbInterfaceType(intf.getInterfaceType()),
1192                         src, dst);
1193             } catch (Exception e) {
1194                 LOG.error("handleInterfaceDelete: failed to delete tunnel port", e);
1195             }
1196         } else if (phyIfName.contains(intf.getName())) {
1197             deletePhysicalPort(srcNode, intf.getName());
1198         } else {
1199             // delete all other interfaces
1200             removeLocalRules(network.getProviderNetworkType(), network.getProviderSegmentationID(),
1201                     srcNode, intf);
1202
1203             if (isVlan(network.getProviderNetworkType())) {
1204                 removeVlanRules(network, srcNode, intf, isLastInstanceOnNode);
1205             } else if (isTunnel(network.getProviderNetworkType())) {
1206
1207                 for (Node dstNode : nodes.values()) {
1208                     InetAddress src = configurationService.getTunnelEndPoint(srcNode);
1209                     InetAddress dst = configurationService.getTunnelEndPoint(dstNode);
1210                     if ((src != null) && (dst != null)) {
1211                         LOG.info("Remove tunnel rules for interface "
1212                                 + intf.getName() + " on srcNode " + srcNode.getNodeId().getValue());
1213                         removeTunnelRules(tunnelType, network.getProviderSegmentationID(),
1214                                 dst, srcNode, intf, true, isLastInstanceOnNode);
1215                         Node dstBridgeNode = southbound.getBridgeNode(dstNode, Constants.INTEGRATION_BRIDGE);
1216                         if(dstBridgeNode != null){
1217                             LOG.info("Remove tunnel rules for interface "
1218                                     + intf.getName() + " on dstNode " + dstNode.getNodeId().getValue());
1219                             removeTunnelRules(tunnelType, network.getProviderSegmentationID(),
1220                                     src, dstBridgeNode, intf, false, isLastInstanceOnNode);
1221                         }
1222                     } else {
1223                         LOG.warn("Tunnel end-point configuration missing. Please configure it in "
1224                                         + "OpenVSwitch Table. "
1225                                         + "Check source {} or destination {}",
1226                                 src != null ? src.getHostAddress() : "null",
1227                                 dst != null ? dst.getHostAddress() : "null");
1228                     }
1229                 }
1230             }
1231         }
1232         return true;
1233     }
1234
1235     @Override
1236     public void initializeFlowRules(Node node) {
1237         initializeFlowRules(node, configurationService.getIntegrationBridgeName());
1238         initializeFlowRules(node, configurationService.getExternalBridgeName());
1239         triggerInterfaceUpdates(node);
1240     }
1241
1242     private void initializeFlowRules(Node node, String bridgeName) {
1243         Long dpid = southbound.getDataPathId(node);
1244         String datapathId = southbound.getDatapathId(node);
1245         LOG.info("initializeFlowRules: bridgeName: {}, dpid: {} - {}",
1246                 bridgeName, dpid, datapathId);
1247
1248         if (dpid == 0L) {
1249             LOG.debug("Openflow Datapath-ID not set for the integration bridge in {}", node);
1250             return;
1251         }
1252
1253         /*
1254          * Table(0) Rule #1
1255          * ----------------
1256          * Match: LLDP (0x88CCL)
1257          * Action: Packet_In to Controller Reserved Port
1258          */
1259
1260         writeLLDPRule(dpid);
1261
1262         if (bridgeName.equals(configurationService.getIntegrationBridgeName()) &&
1263                 NetvirtProvidersProvider.getTableOffset() != 0) {
1264             classifierProvider.programGotoTable(dpid,true);
1265         }
1266
1267         if (bridgeName.equals(configurationService.getExternalBridgeName())) {
1268             writeNormalRule(dpid);
1269         }
1270     }
1271
1272     /*
1273      * Create an LLDP Flow Rule to encapsulate into
1274      * a packet_in that is sent to the controller
1275      * for topology handling.
1276      * Match: Ethertype 0x88CCL
1277      * Action: Punt to Controller in a Packet_In msg
1278      */
1279
1280     private void writeLLDPRule(Long dpidLong) {
1281         classifierProvider.programLLDPPuntRule(dpidLong);
1282     }
1283
1284     /*
1285      * Create a NORMAL Table Miss Flow Rule
1286      * Match: any
1287      * Action: forward to NORMAL pipeline
1288      */
1289
1290     private void writeNormalRule(Long dpidLong) {
1291         NodeBuilder nodeBuilder = FlowUtils.createNodeBuilder(dpidLong);
1292         FlowBuilder flowBuilder = new FlowBuilder();
1293         String flowName = "NORMAL";
1294         FlowUtils.initFlowBuilder(flowBuilder, flowName, Service.CLASSIFIER.getTable()).setPriority(0);
1295         MatchBuilder matchBuilder = new MatchBuilder();
1296         flowBuilder.setMatch(matchBuilder.build());
1297
1298         // Create the OF Actions and Instructions
1299         InstructionBuilder ib = new InstructionBuilder();
1300         InstructionsBuilder isb = new InstructionsBuilder();
1301
1302         // Instructions List Stores Individual Instructions
1303         List<Instruction> instructions = Lists.newArrayList();
1304
1305         // Call the InstructionBuilder Methods Containing Actions
1306         InstructionUtils.createNormalInstructions(FlowUtils.getNodeName(dpidLong), ib);
1307         ib.setOrder(0);
1308         ib.setKey(new InstructionKey(0));
1309         instructions.add(ib.build());
1310
1311         // Add InstructionBuilder to the Instruction(s)Builder List
1312         isb.setInstruction(instructions);
1313
1314         // Add InstructionsBuilder to FlowBuilder
1315         flowBuilder.setInstructions(isb.build());
1316         writeFlow(flowBuilder, nodeBuilder);
1317     }
1318
1319     /*
1320      * (Table:0) Ingress Tunnel Traffic
1321      * Match: OpenFlow InPort and Tunnel ID
1322      * Action: GOTO Local Table (10)
1323      * table=0,tun_id=0x5,in_port=10, actions=goto_table:2
1324      */
1325
1326     private void handleTunnelIn(Long dpidLong, Short writeTable,
1327             Short goToTableId, String segmentationId,
1328             Long ofPort, boolean write) {
1329         classifierProvider.programTunnelIn(dpidLong, segmentationId, ofPort, write);
1330     }
1331
1332     /*
1333      * (Table:0) Ingress VLAN Traffic
1334      * Match: OpenFlow InPort and vlan ID
1335      * Action: GOTO Local Table (20)
1336      * table=0,vlan_id=0x5,in_port=10, actions=goto_table:2
1337      */
1338
1339     private void handleVlanIn(Long dpidLong, Short writeTable, Short goToTableId,
1340             String segmentationId,  Long ethPort, boolean write) {
1341         classifierProvider.programVlanIn(dpidLong, segmentationId, ethPort, write);
1342     }
1343
1344     /*
1345      * (Table:0) Egress VM Traffic Towards TEP
1346      * Match: Destination Ethernet Addr and OpenFlow InPort
1347      * Instruction: Set TunnelID and GOTO Table Tunnel Table (n)
1348      * table=0,in_port=2,dl_src=00:00:00:00:00:01 \
1349      * actions=set_field:5->tun_id,goto_table=1"
1350      */
1351
1352     private void handleLocalInPort(Long dpidLong, Short writeTable, Short goToTableId,
1353             String segmentationId, Long inPort, String attachedMac,
1354             boolean write) {
1355         classifierProvider.programLocalInPort(dpidLong, segmentationId, inPort, attachedMac, write);
1356     }
1357
1358     /*
1359      * (Table:0) Egress VM Traffic Towards TEP
1360      * Match: Source Ethernet Addr and OpenFlow InPort
1361      * Instruction: Set VLANID and GOTO Table Egress (n)
1362      * table=0,in_port=2,dl_src=00:00:00:00:00:01 \
1363      * actions=push_vlan, set_field:5->vlan_id,goto_table=1"
1364      */
1365
1366     private void handleLocalInPortSetVlan(Long dpidLong, Short writeTable,
1367             Short goToTableId, String segmentationId,
1368             Long inPort, String attachedMac,
1369             boolean write) {
1370         classifierProvider.programLocalInPortSetVlan(dpidLong, segmentationId, inPort, attachedMac, write);
1371     }
1372
1373     /*
1374      * (Table:0) Drop frames source from a VM that do not
1375      * match the associated MAC address of the local VM.
1376      * Match: Low priority anything not matching the VM SMAC
1377      * Instruction: Drop
1378      * table=0,priority=16384,in_port=1 actions=drop"
1379      */
1380
1381     private void handleDropSrcIface(Long dpidLong, Long inPort, boolean write) {
1382         classifierProvider.programDropSrcIface(dpidLong, inPort, write);
1383     }
1384
1385     /*
1386      * (Table:1) Egress Tunnel Traffic
1387      * Match: Destination Ethernet Addr and Local InPort
1388      * Instruction: Set TunnelID and GOTO Table Tunnel Table (n)
1389      * table=1,tun_id=0x5,dl_dst=00:00:00:00:00:08 \
1390      * actions=output:10,goto_table:2"
1391      */
1392     private void handleTunnelOut(Long dpidLong, Short writeTable,
1393             Short goToTableId, String segmentationId,
1394             Long OFPortOut, String attachedMac,
1395             boolean write) {
1396         l2ForwardingProvider.programTunnelOut(dpidLong, segmentationId, OFPortOut, attachedMac, write);
1397     }
1398
1399     /*
1400      * (Table:1) Egress VLAN Traffic
1401      * Match: Destination Ethernet Addr and VLAN id
1402      * Instruction: GOTO Table Table 2
1403      * table=1,vlan_id=0x5,dl_dst=00:00:00:00:00:08 \
1404      * actions= goto_table:2"
1405      */
1406     // TODO This method is referenced from commented code above (which needs to be checked)
1407     @SuppressWarnings("unused")
1408     private void handleVlanOut(Long dpidLong, Short writeTable,
1409             Short goToTableId, String segmentationId,
1410             Long ethPort, String attachedMac, boolean write) {
1411         l2ForwardingProvider.programVlanOut(dpidLong, segmentationId, ethPort, attachedMac, write);
1412     }
1413
1414     /*
1415      * (Table:1) Egress Tunnel Traffic
1416      * Match: Destination Ethernet Addr and Local InPort
1417      * Instruction: Set TunnelID and GOTO Table Tunnel Table (n)
1418      * table=1,priority=16384,tun_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
1419      * actions=output:10,output:11,goto_table:2
1420      */
1421
1422     private void handleTunnelFloodOut(Long dpidLong, Short writeTable,
1423             Short localTable, String segmentationId,
1424             Long OFPortOut, boolean write) {
1425         l2ForwardingProvider.programTunnelFloodOut(dpidLong, segmentationId, OFPortOut, write);
1426     }
1427
1428     /*
1429      * (Table:1) Egress VLAN Traffic
1430      * Match: Destination Ethernet Addr and VLAN id
1431      * Instruction: GOTO table 2 and Output port eth interface
1432      * Example: table=1,priority=16384,vlan_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
1433      * actions=output:eth1,goto_table:2
1434      */
1435     // TODO This method is referenced from commented code above (which needs to be checked)
1436     @SuppressWarnings("unused")
1437     private void handleVlanFloodOut(Long dpidLong, Short writeTable,
1438             Short localTable, String segmentationId,
1439             Long localPort, Long ethPort, boolean write) {
1440         //l2ForwardingProvider.programVlanFloodOut(dpidLong, segmentationId, localPort, ethPort, write);
1441     }
1442
1443     /*
1444      * (Table:1) Table Drain w/ Catch All
1445      * Match: Tunnel ID
1446      * Action: GOTO Local Table (10)
1447      * table=2,priority=8192,tun_id=0x5 actions=drop
1448      */
1449
1450     private void handleTunnelMiss(Long dpidLong, Short writeTable,
1451             Short goToTableId, String segmentationId,
1452             boolean write) {
1453         l2ForwardingProvider.programTunnelMiss(dpidLong, segmentationId, write);
1454     }
1455
1456
1457     /*
1458      * (Table:1) Table Drain w/ Catch All
1459      * Match: Vlan ID
1460      * Action: Output port eth interface
1461      * table=1,priority=8192,vlan_id=0x5 actions= output port:eth1
1462      * table=110,priority=8192,dl_vlan=2001 actions=output:2
1463      */
1464
1465     private void handleVlanMiss(Long dpidLong, Short writeTable,
1466             Short goToTableId, String segmentationId,
1467             Long ethPort, boolean write) {
1468         l2ForwardingProvider.programVlanMiss(dpidLong, segmentationId, ethPort, write);
1469     }
1470
1471     /*
1472      * (Table:1) Local Broadcast Flood
1473      * Match: Tunnel ID and dMAC
1474      * Action: Output Port
1475      * table=2,tun_id=0x5,dl_dst=00:00:00:00:00:01 actions=output:2
1476      */
1477
1478     private void handleLocalUcastOut(Long dpidLong, Short writeTable,
1479             String segmentationId, Long localPort,
1480             String attachedMac, boolean write) {
1481         l2ForwardingProvider.programLocalUcastOut(dpidLong, segmentationId, localPort, attachedMac, write);
1482     }
1483
1484     /*
1485      * (Table:2) Local VLAN unicast
1486      * Match: VLAN ID and dMAC
1487      * Action: Output Port
1488      * table=2,vlan_id=0x5,dl_dst=00:00:00:00:00:01 actions=output:2
1489      */
1490
1491     private void handleLocalVlanUcastOut(Long dpidLong, Short writeTable,
1492             String segmentationId, Long localPort,
1493             String attachedMac, boolean write) {
1494         l2ForwardingProvider.programLocalVlanUcastOut(dpidLong, segmentationId, localPort, attachedMac, write);
1495     }
1496
1497     /*
1498      * (Table:2) Local Broadcast Flood
1499      * Match: Tunnel ID and dMAC (::::FF:FF)
1500      * table=2,priority=16384,tun_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
1501      * actions=output:2,3,4,5
1502      */
1503
1504     private void handleLocalBcastOut(Long dpidLong, Short writeTable,
1505             String segmentationId, Long localPort,
1506             boolean write) {
1507         l2ForwardingProvider.programLocalBcastOut(dpidLong, segmentationId, localPort, write);
1508     }
1509
1510     /*
1511      * (Table:2) Local VLAN Broadcast Flood
1512      * Match: vlan ID and dMAC (::::FF:FF)
1513      * table=2,priority=16384,vlan_id=0x5,dl_dst=ff:ff:ff:ff:ff:ff \
1514      * actions=strip_vlan, output:2,3,4,5
1515      * table=110,dl_vlan=2001,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=output:2,pop_vlan,output:1,output:3,output:4
1516      */
1517
1518     private void handleLocalVlanBcastOut(Long dpidLong, Short writeTable, String segmentationId,
1519                                          Long localPort, Long ethPort, boolean write) {
1520         l2ForwardingProvider.programLocalVlanBcastOut(dpidLong, segmentationId, localPort, ethPort, write);
1521     }
1522
1523     /*
1524      * (Table:1) Local Table Miss
1525      * Match: Any Remaining Flows w/a TunID
1526      * Action: Drop w/ a low priority
1527      * table=2,priority=8192,tun_id=0x5 actions=drop
1528      */
1529
1530     private void handleLocalTableMiss(Long dpidLong, Short writeTable,
1531             String segmentationId, boolean write) {
1532         l2ForwardingProvider.programLocalTableMiss(dpidLong, segmentationId, write);
1533     }
1534
1535     /*
1536      * (Table:1) Local Table Miss
1537      * Match: Any Remaining Flows w/a VLAN ID
1538      * Action: Drop w/ a low priority
1539      * table=2,priority=8192,vlan_id=0x5 actions=drop
1540      */
1541     // TODO This method is referenced from commented code above (which needs to be checked)
1542     @SuppressWarnings("unused")
1543     private void handleLocalVlanTableMiss(Long dpidLong, Short writeTable,
1544             String segmentationId, boolean write) {
1545         l2ForwardingProvider.programLocalVlanTableMiss(dpidLong, segmentationId, write);
1546     }
1547
1548     private Group getGroup(GroupBuilder groupBuilder, NodeBuilder nodeBuilder) {
1549         InstanceIdentifier<Group> path1 = InstanceIdentifier.builder(Nodes.class).child(org.opendaylight.yang.gen.v1.urn.opendaylight.inventory
1550                 .rev130819.nodes.Node.class, nodeBuilder.getKey()).augmentation(FlowCapableNode.class).child(Group.class,
1551                         new GroupKey(groupBuilder.getGroupId())).build();
1552         ReadOnlyTransaction readTx = dataBroker.newReadOnlyTransaction();
1553         try {
1554             Optional<Group> data = readTx.read(LogicalDatastoreType.CONFIGURATION, path1).get();
1555             if (data.isPresent()) {
1556                 return data.get();
1557             }
1558         } catch (InterruptedException|ExecutionException e) {
1559             LOG.error("Failed to get group {}", groupBuilder.getGroupName(), e);
1560         }
1561
1562         LOG.debug("Cannot find data for Group {}", groupBuilder.getGroupName());
1563         return null;
1564     }
1565
1566     private void writeGroup(GroupBuilder groupBuilder, NodeBuilder nodeBuilder) {
1567         if (NetvirtProvidersProvider.isMasterProviderInstance()) {
1568             ReadWriteTransaction modification = dataBroker.newReadWriteTransaction();
1569             InstanceIdentifier<Group> path1 = InstanceIdentifier.builder(Nodes.class).child(org.opendaylight.yang.gen.v1.urn.opendaylight.inventory
1570                     .rev130819.nodes.Node.class, nodeBuilder.getKey()).augmentation(FlowCapableNode.class).child(Group.class,
1571                             new GroupKey(groupBuilder.getGroupId())).build();
1572             modification.put(LogicalDatastoreType.CONFIGURATION, path1, groupBuilder.build(), true /*createMissingParents*/);
1573
1574             CheckedFuture<Void, TransactionCommitFailedException> commitFuture = modification.submit();
1575             try {
1576                 commitFuture.get();  // TODO: Make it async (See bug 1362)
1577                 LOG.debug("Transaction success for write of Group {}", groupBuilder.getGroupName());
1578             } catch (InterruptedException|ExecutionException e) {
1579                 LOG.error("Failed to write group {}", groupBuilder.getGroupName(), e);
1580             }
1581         }
1582     }
1583
1584     private void removeGroup(GroupBuilder groupBuilder, NodeBuilder nodeBuilder) {
1585         if (NetvirtProvidersProvider.isMasterProviderInstance()) {
1586             WriteTransaction modification = dataBroker.newWriteOnlyTransaction();
1587             InstanceIdentifier<Group> path1 = InstanceIdentifier.builder(Nodes.class).child(org.opendaylight.yang.gen.v1.urn.opendaylight.inventory
1588                     .rev130819.nodes.Node.class, nodeBuilder.getKey()).augmentation(FlowCapableNode.class).child(Group.class,
1589                             new GroupKey(groupBuilder.getGroupId())).build();
1590             modification.delete(LogicalDatastoreType.CONFIGURATION, path1);
1591             CheckedFuture<Void, TransactionCommitFailedException> commitFuture = modification.submit();
1592
1593             try {
1594                 commitFuture.get();  // TODO: Make it async (See bug 1362)
1595                 LOG.debug("Transaction success for deletion of Group {}", groupBuilder.getGroupName());
1596             } catch (InterruptedException|ExecutionException e) {
1597                 LOG.error("Failed to remove group {}", groupBuilder.getGroupName(), e);
1598             }
1599         }
1600     }
1601
1602     private void writeFlow(FlowBuilder flowBuilder, NodeBuilder nodeBuilder) {
1603         if (NetvirtProvidersProvider.isMasterProviderInstance()){
1604             ReadWriteTransaction modification = dataBroker.newReadWriteTransaction();
1605             InstanceIdentifier<Flow> path1 =
1606                     InstanceIdentifier.builder(Nodes.class).child(org.opendaylight.yang.gen.v1.urn.opendaylight.inventory
1607                                     .rev130819.nodes.Node.class,
1608                             nodeBuilder.getKey()).augmentation(FlowCapableNode.class).child(Table.class,
1609                             new TableKey(flowBuilder.getTableId())).child(Flow.class, flowBuilder.getKey()).build();
1610
1611             //modification.put(LogicalDatastoreType.OPERATIONAL, path1, flowBuilder.build());
1612             modification.put(LogicalDatastoreType.CONFIGURATION, path1, flowBuilder.build(),
1613                     true);//createMissingParents
1614
1615
1616             CheckedFuture<Void, TransactionCommitFailedException> commitFuture = modification.submit();
1617             try {
1618                 commitFuture.get();  // TODO: Make it async (See bug 1362)
1619                 LOG.debug("Transaction success for write of Flow {}", flowBuilder.getFlowName());
1620             } catch (InterruptedException|ExecutionException e) {
1621                 LOG.error("Failed to write flows {}", flowBuilder.getFlowName(), e);
1622             }
1623         }
1624     }
1625
1626     /**
1627      * Create Output Port Group Instruction
1628      *
1629      * @param ib       Map InstructionBuilder without any instructions
1630      * @param dpidLong Long the datapath ID of a switch/node
1631      * @param port     Long representing a port on a switch/node
1632      * @return ib InstructionBuilder Map with instructions
1633      */
1634     // TODO This method is referenced from commented code in L2ForwardingService (which needs to be checked)
1635     @SuppressWarnings("unused")
1636     protected InstructionBuilder createOutputGroupInstructions(NodeBuilder nodeBuilder,
1637             InstructionBuilder ib,
1638             Long dpidLong, Long port ,
1639             List<Instruction> instructions) {
1640         NodeConnectorId ncid = new NodeConnectorId(Constants.OPENFLOW_NODE_PREFIX + dpidLong + ":" + port);
1641         LOG.debug("createOutputGroupInstructions() Node Connector ID is - Type=openflow: DPID={} port={} existingInstructions={}", dpidLong, port, instructions);
1642
1643         List<Action> actionList = Lists.newArrayList();
1644         ActionBuilder ab = new ActionBuilder();
1645
1646         List<Action> existingActions;
1647         if (instructions != null) {
1648             for (Instruction in : instructions) {
1649                 if (in.getInstruction() instanceof ApplyActionsCase) {
1650                     existingActions = (((ApplyActionsCase) in.getInstruction()).getApplyActions().getAction());
1651                     actionList.addAll(existingActions);
1652                 }
1653             }
1654         }
1655
1656         GroupBuilder groupBuilder = new GroupBuilder();
1657         Group group = null;
1658
1659         /* Create output action for this port*/
1660         OutputActionBuilder oab = new OutputActionBuilder();
1661         oab.setOutputNodeConnector(ncid);
1662         ab.setAction(new OutputActionCaseBuilder().setOutputAction(oab.build()).build());
1663         LOG.debug("createOutputGroupInstructions(): output action {}", ab.build());
1664         boolean addNew = true;
1665         boolean groupActionAdded = false;
1666
1667         /* Find the group action and get the group */
1668         for (Action action : actionList) {
1669             if (action.getAction() instanceof GroupActionCase) {
1670                 groupActionAdded = true;
1671                 GroupActionCase groupAction = (GroupActionCase) action.getAction();
1672                 Long id = groupAction.getGroupAction().getGroupId();
1673                 String groupName = groupAction.getGroupAction().getGroup();
1674                 GroupKey key = new GroupKey(new GroupId(id));
1675
1676                 groupBuilder.setGroupId(new GroupId(id));
1677                 groupBuilder.setGroupName(groupName);
1678                 groupBuilder.setGroupType(GroupTypes.GroupAll);
1679                 groupBuilder.setKey(key);
1680                 group = getGroup(groupBuilder, nodeBuilder);
1681                 LOG.debug("createOutputGroupInstructions: group {}", group);
1682                 break;
1683             }
1684         }
1685
1686         LOG.debug("createOutputGroupInstructions: groupActionAdded {}", groupActionAdded);
1687         if (groupActionAdded) {
1688             /* modify the action bucket in group */
1689             groupBuilder = new GroupBuilder(group);
1690             Buckets buckets = groupBuilder.getBuckets();
1691             for (Bucket bucket : buckets.getBucket()) {
1692                 List<Action> bucketActions = bucket.getAction();
1693                 LOG.debug("createOutputGroupInstructions: bucketActions {}", bucketActions);
1694                 for (Action action : bucketActions) {
1695                     if (action.getAction() instanceof OutputActionCase) {
1696                         OutputActionCase opAction = (OutputActionCase)action.getAction();
1697                         /* If output port action already in the action list of one of the buckets, skip */
1698                         if (opAction.getOutputAction().getOutputNodeConnector().equals(new Uri(ncid))) {
1699                             addNew = false;
1700                             break;
1701                         }
1702                     }
1703                 }
1704             }
1705             LOG.debug("createOutputGroupInstructions: addNew {}", addNew);
1706             if (addNew && !buckets.getBucket().isEmpty()) {
1707                 /* the new output action is not in the bucket, add to bucket */
1708                 Bucket bucket = buckets.getBucket().get(0);
1709                 List<Action> bucketActionList = Lists.newArrayList();
1710                 bucketActionList.addAll(bucket.getAction());
1711                 /* set order for new action and add to action list */
1712                 ab.setOrder(bucketActionList.size());
1713                 ab.setKey(new ActionKey(bucketActionList.size()));
1714                 bucketActionList.add(ab.build());
1715
1716                 /* set bucket and buckets list. Reset groupBuilder with new buckets.*/
1717                 BucketsBuilder bucketsBuilder = new BucketsBuilder();
1718                 List<Bucket> bucketList = Lists.newArrayList();
1719                 BucketBuilder bucketBuilder = new BucketBuilder();
1720                 bucketBuilder.setBucketId(new BucketId((long) 1));
1721                 bucketBuilder.setKey(new BucketKey(new BucketId((long) 1)));
1722                 bucketBuilder.setAction(bucketActionList);
1723                 bucketList.add(bucketBuilder.build());
1724                 bucketsBuilder.setBucket(bucketList);
1725                 groupBuilder.setBuckets(bucketsBuilder.build());
1726                 LOG.debug("createOutputGroupInstructions: bucketList {}", bucketList);
1727             }
1728         } else {
1729             /* create group */
1730             groupBuilder = new GroupBuilder();
1731             groupBuilder.setGroupType(GroupTypes.GroupAll);
1732             groupBuilder.setGroupId(new GroupId(groupId));
1733             groupBuilder.setKey(new GroupKey(new GroupId(groupId)));
1734             groupBuilder.setGroupName("Output port group " + groupId);
1735             groupBuilder.setBarrier(false);
1736
1737             BucketsBuilder bucketBuilder = new BucketsBuilder();
1738             List<Bucket> bucketList = Lists.newArrayList();
1739             BucketBuilder bucket = new BucketBuilder();
1740             bucket.setBucketId(new BucketId((long) 1));
1741             bucket.setKey(new BucketKey(new BucketId((long) 1)));
1742
1743             /* put output action to the bucket */
1744             List<Action> bucketActionList = Lists.newArrayList();
1745             /* set order for new action and add to action list */
1746             ab.setOrder(bucketActionList.size());
1747             ab.setKey(new ActionKey(bucketActionList.size()));
1748             bucketActionList.add(ab.build());
1749
1750             bucket.setAction(bucketActionList);
1751             bucketList.add(bucket.build());
1752             bucketBuilder.setBucket(bucketList);
1753             groupBuilder.setBuckets(bucketBuilder.build());
1754
1755             /* Add new group action */
1756             GroupActionBuilder groupActionB = new GroupActionBuilder();
1757             groupActionB.setGroupId(groupId);
1758             groupActionB.setGroup("Output port group " + groupId);
1759             ab = new ActionBuilder();
1760             ab.setAction(new GroupActionCaseBuilder().setGroupAction(groupActionB.build()).build());
1761             ab.setOrder(actionList.size());
1762             ab.setKey(new ActionKey(actionList.size()));
1763             actionList.add(ab.build());
1764
1765             groupId++;
1766         }
1767         LOG.debug("createOutputGroupInstructions: group {}", groupBuilder.build());
1768         LOG.debug("createOutputGroupInstructions: actionList {}", actionList);
1769
1770         if (addNew) {
1771             /* rewrite the group to group table */
1772             writeGroup(groupBuilder, nodeBuilder);
1773         }
1774
1775         // Create an Apply Action
1776         ApplyActionsBuilder aab = new ApplyActionsBuilder();
1777         aab.setAction(actionList);
1778         ib.setInstruction(new ApplyActionsCaseBuilder().setApplyActions(aab.build()).build());
1779
1780         return ib;
1781     }
1782
1783     /**
1784      * Remove Output Port from action list in group bucket
1785      *
1786      * @param ib       Map InstructionBuilder without any instructions
1787      * @param dpidLong Long the datapath ID of a switch/node
1788      * @param port     Long representing a port on a switch/node
1789      * @return ib InstructionBuilder Map with instructions
1790      */
1791     // TODO This method is referenced from commented code in L2ForwardingService (which needs to be checked)
1792     @SuppressWarnings("unused")
1793     protected boolean removeOutputPortFromGroup(NodeBuilder nodeBuilder, InstructionBuilder ib,
1794             Long dpidLong, Long port , List<Instruction> instructions) {
1795
1796         NodeConnectorId ncid = new NodeConnectorId(Constants.OPENFLOW_NODE_PREFIX + dpidLong + ":" + port);
1797         LOG.debug("removeOutputPortFromGroup() Node Connector ID is - Type=openflow: DPID={} port={} existingInstructions={}", dpidLong, port, instructions);
1798
1799         List<Action> actionList = Lists.newArrayList();
1800         ActionBuilder ab;
1801
1802         List<Action> existingActions;
1803         if (instructions != null) {
1804             for (Instruction in : instructions) {
1805                 if (in.getInstruction() instanceof ApplyActionsCase) {
1806                     existingActions = (((ApplyActionsCase) in.getInstruction()).getApplyActions().getAction());
1807                     actionList.addAll(existingActions);
1808                     break;
1809                 }
1810             }
1811         }
1812
1813         GroupBuilder groupBuilder = new GroupBuilder();
1814         Group group = null;
1815         boolean groupActionAdded = false;
1816         /* Find the group action and get the group */
1817         for (Action action : actionList) {
1818             if (action.getAction() instanceof GroupActionCase) {
1819                 groupActionAdded = true;
1820                 GroupActionCase groupAction = (GroupActionCase) action.getAction();
1821                 Long id = groupAction.getGroupAction().getGroupId();
1822                 String groupName = groupAction.getGroupAction().getGroup();
1823                 GroupKey key = new GroupKey(new GroupId(id));
1824
1825                 groupBuilder.setGroupId(new GroupId(id));
1826                 groupBuilder.setGroupName(groupName);
1827                 groupBuilder.setGroupType(GroupTypes.GroupAll);
1828                 groupBuilder.setKey(key);
1829                 group = getGroup(groupBuilder, nodeBuilder);
1830                 break;
1831             }
1832         }
1833
1834         if (groupActionAdded) {
1835             /* modify the action bucket in group */
1836             groupBuilder = new GroupBuilder(group);
1837             Buckets buckets = groupBuilder.getBuckets();
1838             List<Action> bucketActions = Lists.newArrayList();
1839             for (Bucket bucket : buckets.getBucket()) {
1840                 int index = 0;
1841                 boolean isPortDeleted = false;
1842                 bucketActions = bucket.getAction();
1843                 for (Action action : bucketActions) {
1844                     if (action.getAction() instanceof OutputActionCase) {
1845                         OutputActionCase opAction = (OutputActionCase)action.getAction();
1846                         if (opAction.getOutputAction().getOutputNodeConnector().equals(new Uri(ncid))) {
1847                             /* Find the output port in action list and remove */
1848                             index = bucketActions.indexOf(action);
1849                             bucketActions.remove(action);
1850                             isPortDeleted = true;
1851                             break;
1852                         }
1853                     }
1854                 }
1855                 if (isPortDeleted && !bucketActions.isEmpty()) {
1856                     for (int i = index; i< bucketActions.size(); i++) {
1857                         Action action = bucketActions.get(i);
1858                         if (action.getOrder() != i) {
1859                             /* Shift the action order */
1860                             ab = new ActionBuilder();
1861                             ab.setAction(action.getAction());
1862                             ab.setOrder(i);
1863                             ab.setKey(new ActionKey(i));
1864                             Action actionNewOrder = ab.build();
1865                             bucketActions.remove(action);
1866                             bucketActions.add(i, actionNewOrder);
1867                         }
1868                     }
1869
1870                 } else if (bucketActions.isEmpty()) {
1871                     /* remove bucket with empty action list */
1872                     buckets.getBucket().remove(bucket);
1873                     break;
1874                 }
1875             }
1876             if (!buckets.getBucket().isEmpty()) {
1877                 /* rewrite the group to group table */
1878                 /* set bucket and buckets list. Reset groupBuilder with new buckets.*/
1879                 BucketsBuilder bucketsBuilder = new BucketsBuilder();
1880                 List<Bucket> bucketList = Lists.newArrayList();
1881                 BucketBuilder bucketBuilder = new BucketBuilder();
1882                 bucketBuilder.setBucketId(new BucketId((long) 1));
1883                 bucketBuilder.setKey(new BucketKey(new BucketId((long) 1)));
1884                 bucketBuilder.setAction(bucketActions);
1885                 bucketList.add(bucketBuilder.build());
1886                 bucketsBuilder.setBucket(bucketList);
1887                 groupBuilder.setBuckets(bucketsBuilder.build());
1888                 LOG.debug("removeOutputPortFromGroup: bucketList {}", bucketList);
1889
1890                 writeGroup(groupBuilder, nodeBuilder);
1891                 ApplyActionsBuilder aab = new ApplyActionsBuilder();
1892                 aab.setAction(actionList);
1893                 ib.setInstruction(new ApplyActionsCaseBuilder().setApplyActions(aab.build()).build());
1894                 return false;
1895             } else {
1896                 /* remove group with empty bucket. return true to delete flow */
1897                 removeGroup(groupBuilder, nodeBuilder);
1898                 return true;
1899             }
1900         } else {
1901             /* no group for port list. flow can be removed */
1902             return true;
1903         }
1904     }
1905
1906     @Override
1907     public void initializeOFFlowRules(Node openflowNode) {
1908         String bridgeName = southbound.getBridgeName(openflowNode);
1909         LOG.info("initializeOFFlowRules: bridgeName: {}", bridgeName);
1910         if (bridgeName.equals(configurationService.getIntegrationBridgeName())) {
1911             initializeFlowRules(openflowNode, configurationService.getIntegrationBridgeName());
1912             triggerInterfaceUpdates(openflowNode);
1913         } else if (bridgeName.equals(configurationService.getExternalBridgeName())) {
1914             initializeFlowRules(openflowNode, configurationService.getExternalBridgeName());
1915             LOG.info("initializeOFFlowRules after writeFlow: bridgeName: {}", bridgeName);
1916             triggerInterfaceUpdates(openflowNode);
1917             LOG.info("initializeOFFlowRules after triggerUpdates: bridgeName: {}", bridgeName);
1918         }
1919     }
1920
1921     public static NodeBuilder createNodeBuilder(String nodeId) {
1922         NodeBuilder builder = new NodeBuilder();
1923         builder.setId(new NodeId(nodeId));
1924         builder.setKey(new NodeKey(builder.getId()));
1925         return builder;
1926     }
1927
1928     @Override
1929     public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
1930         this.bundleContext = bundleContext;
1931         configurationService =
1932                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1933         tenantNetworkManager =
1934                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1935         bridgeConfigurationManager =
1936                 (BridgeConfigurationManager) ServiceHelper.getGlobalInstance(BridgeConfigurationManager.class, this);
1937         nodeCacheManager =
1938                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1939         classifierProvider =
1940                 (ClassifierProvider) ServiceHelper.getGlobalInstance(ClassifierProvider.class, this);
1941         ingressAclProvider =
1942                 (IngressAclProvider) ServiceHelper.getGlobalInstance(IngressAclProvider.class, this);
1943         egressAclProvider =
1944                 (EgressAclProvider) ServiceHelper.getGlobalInstance(EgressAclProvider.class, this);
1945         l2ForwardingProvider =
1946                 (L2ForwardingProvider) ServiceHelper.getGlobalInstance(L2ForwardingProvider.class, this);
1947         securityServicesManager =
1948                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1949         southbound =
1950                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1951     }
1952
1953     @Override
1954     public void setDependencies(Object impl) {
1955         if (impl instanceof NetworkingProviderManager) {
1956             NetworkingProviderManager networkingProviderManager = (NetworkingProviderManager) impl;
1957             networkingProviderManager.providerAdded(
1958                     bundleContext.getServiceReference(NetworkingProvider.class.getName()), this);
1959         }
1960     }
1961 }