git.opendaylight Code Review - netvirt.git/blob

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlan;
  11 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.base.Preconditions;
  15 import com.google.common.collect.Lists;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import java.math.BigInteger;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.HashSet;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.Queue;
  25 import java.util.Set;
  26 import java.util.concurrent.ConcurrentHashMap;
  27 import java.util.concurrent.ConcurrentLinkedQueue;
  28 import java.util.concurrent.ConcurrentMap;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.Nonnull;
  31 import javax.annotation.PostConstruct;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.apache.commons.lang3.StringUtils;
  35 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  36 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  37 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  38 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  39 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  40 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  41 import org.opendaylight.genius.itm.globals.ITMConstants;
  42 import org.opendaylight.genius.mdsalutil.FlowEntity;
  43 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  44 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  45 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  46 import org.opendaylight.genius.mdsalutil.MatchInfo;
  47 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  48 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  49 import org.opendaylight.genius.mdsalutil.NwConstants;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  55 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  56 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  57 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  58 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  59 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  60 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  61 import org.opendaylight.genius.utils.ServiceIndex;
  62 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  63 import org.opendaylight.netvirt.elan.ElanException;
  64 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  65 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  66 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  67 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  68 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  69 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  70 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  71 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  72 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  73 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  74 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  75 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  76 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  77 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  78 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.elan.instance.ExternalTeps;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 120 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 122 import org.slf4j.Logger;
 123 import org.slf4j.LoggerFactory;
 124
 125 /**
 126  * Class in charge of handling creations, modifications and removals of
 127  * ElanInterfaces.
 128  *
 129  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 130  */
 131 @Singleton
 132 @SuppressWarnings("deprecation")
 133 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager> {
 134
 135     private final DataBroker broker;
 136     private final IMdsalApiManager mdsalManager;
 137     private final IInterfaceManager interfaceManager;
 138     private final IdManagerService idManager;
 139     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 140     private final INeutronVpnManager neutronVpnManager;
 141     private final ElanItmUtils elanItmUtils;
 142     private final ElanEtreeUtils elanEtreeUtils;
 143     private final ElanL2GatewayUtils elanL2GatewayUtils;
 144     private ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 145     private ElanUtils elanUtils;
 146     private final JobCoordinator jobCoordinator;
 147
 148     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 149     private static final boolean SH_FLAG_SET = true;
 150     private static final boolean SH_FLAG_UNSET = false;
 151
 152     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 153         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 154
 155     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 156
 157     @Inject
 158     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 159                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 160                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 161                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 162                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 163                                 final JobCoordinator jobCoordinator) {
 164         super(ElanInterface.class, ElanInterfaceManager.class);
 165         this.broker = dataBroker;
 166         this.idManager = managerService;
 167         this.mdsalManager = mdsalApiManager;
 168         this.interfaceManager = interfaceManager;
 169         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 170         this.neutronVpnManager = neutronVpnManager;
 171         this.elanItmUtils = elanItmUtils;
 172         this.elanEtreeUtils = elanEtreeUtils;
 173         this.elanL2GatewayUtils = elanL2GatewayUtils;
 174         this.jobCoordinator = jobCoordinator;
 175     }
 176
 177     /**
 178      * This method is used instead of "regular" standard constructor dependency injection in, only,
 179      * ElanServiceProvider's constructor to wire things together. It was done like this because of the unholy
 180      * triumvirate of unhealthy love triangle between (at least) ElanUtils, ElanInterfaceManager and
 181      * ElanL2GatewayMulticastUtils. The proper solution to be able to get rid of this would be to split up some of these
 182      * relatively big classes into smaller classes, and then inject more fine grained among them.
 183      */
 184     public void setElanUtils(ElanUtils elanUtils) {
 185         this.elanUtils = elanUtils;
 186         this.elanL2GatewayMulticastUtils = elanUtils.getElanL2GatewayMulticastUtils();
 187         this.elanForwardingEntriesHandler.setElanUtils(elanUtils);
 188     }
 189
 190     @Override
 191     @PostConstruct
 192     public void init() {
 193         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 194     }
 195
 196     @Override
 197     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 198         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 199     }
 200
 201     @Override
 202     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 203         String interfaceName = del.getName();
 204         ElanInstance elanInfo = ElanUtils.getElanInstanceByName(broker, del.getElanInstanceName());
 205         /*
 206          * Handling in case the elan instance is deleted.If the Elan instance is
 207          * deleted, there is no need to explicitly delete the elan interfaces
 208          */
 209         if (elanInfo == null) {
 210             return;
 211         }
 212         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 213         if (interfaceInfo == null && elanInfo.isExternal()) {
 214             // In deleting external network, the underlying ietf Inteface might have been removed
 215             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 216             // from Operational DS instead
 217             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 218         }
 219         String elanInstanceName = elanInfo.getElanInstanceName();
 220         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 221                 interfaceName, interfaceInfo, false, this);
 222         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 223     }
 224
 225     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 226             InterfaceInfo interfaceInfo, boolean isInterfaceStateRemoved) {
 227         String elanName = elanInfo.getElanInstanceName();
 228         boolean isLastElanInterface = false;
 229         boolean isLastInterfaceOnDpn = false;
 230         BigInteger dpId = null;
 231         long elanTag = elanInfo.getElanTag();
 232         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 233         WriteTransaction interfaceTx = broker.newWriteOnlyTransaction();
 234         Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 235         if (elanState == null) {
 236             interfaceTx.cancel();
 237             return Collections.emptyList();
 238         }
 239         WriteTransaction flowTx = broker.newWriteOnlyTransaction();
 240         List<String> elanInterfaces = elanState.getElanInterfaces();
 241         if (elanInterfaces.isEmpty()) {
 242             isLastElanInterface = true;
 243         }
 244         if (interfaceInfo != null) {
 245             dpId = interfaceInfo.getDpId();
 246             DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, dpId,
 247                     interfaceName, elanTag, interfaceTx);
 248             /*
 249              * If there are not elan ports, remove the unknown dmac, terminating
 250              * service table flows, remote/local bc group
 251              */
 252             if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 253                     || dpnInterfaces.getInterfaces().isEmpty()) {
 254                 // No more Elan Interfaces in this DPN
 255                 LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(), dpId);
 256                 if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 257                     removeDefaultTermFlow(dpId, elanInfo.getElanTag());
 258                 }
 259                 removeUnknownDmacFlow(dpId, elanInfo, flowTx, elanInfo.getElanTag());
 260                 removeEtreeUnknownDmacFlow(dpId, elanInfo, flowTx);
 261                 removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 262                 removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 263                 removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 264                 if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 265                     if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 266                         elanUtils.removeTerminatingServiceAction(dpId, elanInfo.getSegmentationId().intValue());
 267                     }
 268                     unsetExternalTunnelTable(dpId, elanInfo);
 269                 }
 270                 isLastInterfaceOnDpn = true;
 271             } else {
 272                 setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 273             }
 274         }
 275
 276         List<ListenableFuture<Void>> futures = new ArrayList<>();
 277         futures.add(ElanUtils.waitForTransactionToComplete(interfaceTx));
 278         futures.add(ElanUtils.waitForTransactionToComplete(flowTx));
 279
 280         if (isLastInterfaceOnDpn && dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 281             setElanAndEtreeBCGrouponOtherDpns(elanInfo, dpId);
 282         }
 283         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 284                 interfaceName, elanInfo, interfaceInfo, isInterfaceStateRemoved, this, isLastElanInterface);
 285         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 286                 ElanConstants.JOB_MAX_RETRIES);
 287
 288         return futures;
 289     }
 290
 291     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 292             WriteTransaction deleteFlowGroupTx) {
 293         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 294         if (etreeLeafTag != null) {
 295             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 296             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 297         }
 298     }
 299
 300     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 301             WriteTransaction deleteFlowGroupTx) {
 302         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 303         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 304     }
 305
 306     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 307             WriteTransaction deleteFlowGroupTx) {
 308         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 309         if (etreeInstance != null) {
 310             BigInteger dpnId = interfaceInfo.getDpId();
 311             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 312             List<Bucket> listBuckets = new ArrayList<>();
 313             int bucketId = 0;
 314             listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
 315             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 316                     MDSALUtil.buildBucketLists(listBuckets));
 317             LOG.trace("deleted the localBroadCast Group:{}", group);
 318             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 319         }
 320     }
 321
 322     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 323             WriteTransaction deleteFlowGroupTx) {
 324         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 325         if (etreeInstance != null) {
 326             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 327             int bucketId = 0;
 328             int actionKey = 0;
 329             List<Bucket> listBuckets = new ArrayList<>();
 330             List<Action> listAction = new ArrayList<>();
 331             listAction.add(new ActionGroup(ElanUtils.getEtreeLeafLocalBCGId(etreeTag)).buildAction(++actionKey));
 332             listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
 333                     MDSALUtil.WATCH_GROUP));
 334             bucketId++;
 335             listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, etreeTag));
 336             BigInteger dpnId = interfaceInfo.getDpId();
 337             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 338             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 339                     MDSALUtil.buildBucketLists(listBuckets));
 340             LOG.trace("deleting the remoteBroadCast group:{}", group);
 341             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 342         }
 343     }
 344
 345     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName, WriteTransaction tx) {
 346         String elanName = elanInfo.getElanInstanceName();
 347         Elan elanState = ElanUtils.getElanByName(broker, elanName);
 348         if (elanState == null) {
 349             return elanState;
 350         }
 351         List<String> elanInterfaces = elanState.getElanInterfaces();
 352         boolean isRemoved = elanInterfaces.remove(interfaceName);
 353         if (!isRemoved) {
 354             return elanState;
 355         }
 356
 357         if (elanInterfaces.isEmpty()) {
 358             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName));
 359             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanMacTableOperationalDataPath(elanName));
 360             tx.delete(LogicalDatastoreType.OPERATIONAL,
 361                     ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 362         } else {
 363             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 364                     .setKey(new ElanKey(elanName)).build();
 365             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName),
 366                     updateElanState);
 367         }
 368         return elanState;
 369     }
 370
 371     private void deleteElanInterfaceFromConfigDS(String interfaceName, WriteTransaction tx) {
 372         // removing the ElanInterface from the config data_store if interface is
 373         // not present in Interface config DS
 374         if (interfaceManager.getInterfaceInfoFromConfigDataStore(interfaceName) == null
 375                 && ElanUtils.getElanInterfaceByElanInterfaceName(broker, interfaceName) != null) {
 376             tx.delete(LogicalDatastoreType.CONFIGURATION,
 377                     ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 378         }
 379     }
 380
 381     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 382             interfaceInfo, String interfaceName, boolean isInterfaceStateRemoved, boolean isLastElanInterface) {
 383         String elanName = elanInfo.getElanInstanceName();
 384         WriteTransaction interfaceTx = broker.newWriteOnlyTransaction();
 385         WriteTransaction flowTx = broker.newWriteOnlyTransaction();
 386         InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 387                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 388         Optional<ElanInterfaceMac> existingElanInterfaceMac = ElanUtils.read(broker,
 389                 LogicalDatastoreType.OPERATIONAL, elanInterfaceId);
 390         LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 391         if (interfaceInfo != null) {
 392             if (existingElanInterfaceMac.isPresent()) {
 393                 List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 394                 List<MacEntry> macEntries = new ArrayList<>();
 395                 if (existingMacEntries != null && !existingMacEntries.isEmpty()) {
 396                     macEntries.addAll(existingMacEntries);
 397                 }
 398                 List<PhysAddress> macAddresses = macEntries.stream().map(macEntry -> {
 399                     PhysAddress macAddress = macEntry.getMacAddress();
 400                     LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 401                             macAddress.getValue(), interfaceName);
 402                     Optional<MacEntry> macEntryOptional = elanUtils.getMacEntryForElanInstance(elanName,
 403                             macAddress);
 404                     if (!isLastElanInterface && macEntryOptional.isPresent()) {
 405                         interfaceTx.delete(LogicalDatastoreType.OPERATIONAL,
 406                                 ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 407                     }
 408                     elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 409                     return macAddress;
 410                 }).collect(Collectors.toList());
 411
 412                 // Removing all those MACs from External Devices belonging
 413                 // to this ELAN
 414                 if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 415                     elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 416                 }
 417             }
 418             removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 419             removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 420         } else if (existingElanInterfaceMac.isPresent()) {
 421             // Interface does not exist in ConfigDS, so lets remove everything
 422             // about that interface related to Elan
 423             List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 424             if (macEntries != null) {
 425                 for (MacEntry macEntry : macEntries) {
 426                     PhysAddress macAddress = macEntry.getMacAddress();
 427                     if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 428                         interfaceTx.delete(LogicalDatastoreType.OPERATIONAL,
 429                                 ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 430                     }
 431                 }
 432             }
 433         }
 434         if (existingElanInterfaceMac.isPresent()) {
 435             interfaceTx.delete(LogicalDatastoreType.OPERATIONAL, elanInterfaceId);
 436         }
 437         if (!isInterfaceStateRemoved) {
 438             unbindService(interfaceName, interfaceTx);
 439         }
 440         deleteElanInterfaceFromConfigDS(interfaceName, interfaceTx);
 441         List<ListenableFuture<Void>> futures = new ArrayList<>();
 442         futures.add(ElanUtils.waitForTransactionToComplete(interfaceTx));
 443         futures.add(ElanUtils.waitForTransactionToComplete(flowTx));
 444         return futures;
 445     }
 446
 447     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 448                                                                          String interfaceName, long elanTag,
 449                                                                          WriteTransaction tx) {
 450         DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 451         if (dpnInterfaces != null) {
 452             List<String> interfaceLists = dpnInterfaces.getInterfaces();
 453             if (interfaceLists != null) {
 454                 interfaceLists.remove(interfaceName);
 455             }
 456
 457             if (interfaceLists == null || interfaceLists.isEmpty()) {
 458                 deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 459                 deleteElanDpnInterface(elanName, dpId, tx);
 460             } else {
 461                 dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 462             }
 463         }
 464         return dpnInterfaces;
 465     }
 466
 467     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 468         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 469         for (DpnInterfaces dpnInterface : dpnInterfaces) {
 470             BigInteger currentDpId = dpnInterface.getDpId();
 471             if (!currentDpId.equals(dpId)) {
 472                 for (String elanInterface : dpnInterface.getInterfaces()) {
 473                     ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 474                     if (macs == null || macs.getMacEntry() == null) {
 475                         continue;
 476                     }
 477                     for (MacEntry mac : macs.getMacEntry()) {
 478                         removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 479                         removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 480                     }
 481                 }
 482             }
 483         }
 484     }
 485
 486     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 487         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 488         if (etreeLeafTag != null) {
 489             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac);
 490         }
 491     }
 492
 493     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 494         mdsalManager
 495                 .removeFlow(dpId,
 496                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 497                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 498                                         mac.getMacAddress().getValue(), elanTag)));
 499     }
 500
 501     /*
 502     * Possible Scenarios for update
 503     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 504         then all {1,2,3,4} should be removed
 505
 506         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 507         then all {1,2,3,4} should be added
 508
 509         c. if orig = {1,2,3,4} updated={2,3,4}
 510         then 1 should be removed
 511
 512         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 513         then we should just add 5
 514
 515         e. if orig = {1,2,3,4} updated={2,3,4,5}
 516         then 1 should be removed , 5 should be added
 517     * */
 518     @Override
 519     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 520         // updating the static-Mac Entries for the existing elanInterface
 521         String elanName = update.getElanInstanceName();
 522         String interfaceName = update.getName();
 523
 524         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 525         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 526         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 527         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 528
 529         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 530                 deletedEntry.getMacAddress()));
 531
 532         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 533         * Hence add the macentries for the same.*/
 534         for (StaticMacEntries staticMacEntry : updatedEntries) {
 535             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 536                     staticMacEntry.getMacAddress());
 537             Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 538                     LogicalDatastoreType.OPERATIONAL, macEntryIdentifier);
 539             WriteTransaction tx = broker.newWriteOnlyTransaction();
 540             if (existingMacEntry.isPresent()) {
 541                 elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 542                         elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 543                         tx);
 544             } else {
 545                 elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 546                         ElanUtils.getElanInstanceByName(broker, elanName), interfaceName, staticMacEntry, tx);
 547             }
 548             ElanUtils.waitForTransactionToComplete(tx);
 549         }
 550     }
 551
 552     @Override
 553     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 554         String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 555         String interfaceName = elanInterfaceAdded.getName();
 556         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 557         if (interfaceInfo == null) {
 558             LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 559             return;
 560         }
 561         ElanInstance elanInstance = ElanUtils.getElanInstanceByName(broker, elanInstanceName);
 562
 563         if (elanInstance == null) {
 564             elanInstance = new ElanInstanceBuilder().setElanInstanceName(elanInstanceName)
 565                     .setDescription(elanInterfaceAdded.getDescription()).build();
 566             // Add the ElanInstance in the Configuration data-store
 567             WriteTransaction tx = broker.newWriteOnlyTransaction();
 568             List<String> elanInterfaces = new ArrayList<>();
 569             elanInterfaces.add(interfaceName);
 570             ElanUtils.updateOperationalDataStore(broker, idManager,
 571                     elanInstance, elanInterfaces, tx);
 572             ElanUtils.waitForTransactionToComplete(tx);
 573             elanInstance = ElanUtils.getElanInstanceByName(broker, elanInstanceName);
 574         }
 575
 576         Long elanTag = elanInstance.getElanTag();
 577         // If elan tag is not updated, then put the elan interface into
 578         // unprocessed entry map and entry. Let entries
 579         // in this map get processed during ELAN update DCN.
 580         if (elanTag == null) {
 581             ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 582             if (elanInterfaces == null) {
 583                 elanInterfaces = new ConcurrentLinkedQueue<>();
 584             }
 585             elanInterfaces.add(elanInterfaceAdded);
 586             unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 587             return;
 588         }
 589         InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 590                 interfaceInfo, elanInstance, this);
 591         jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 592     }
 593
 594     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) throws ElanException {
 595         List<ListenableFuture<Void>> futures = new ArrayList<>();
 596         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 597         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 598             return futures;
 599         }
 600         for (ElanInterface elanInterface : elanInterfaces) {
 601             String interfaceName = elanInterface.getName();
 602             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 603             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 604         }
 605         return futures;
 606     }
 607
 608     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 609             WriteTransaction writeFlowGroupTx) throws ElanException {
 610         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 611                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 612         List<DpnInterfaces> dpnInterfaceLists = null;
 613         if (elanDpnInterfacesList != null) {
 614             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 615         }
 616         if (dpnInterfaceLists == null) {
 617             dpnInterfaceLists = new ArrayList<>();
 618         }
 619         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 620             BigInteger dstDpId = interfaceInfo.getDpId();
 621             if (dpnInterfaces.getDpId().equals(dstDpId)) {
 622                 continue;
 623             }
 624             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 625             for (String remoteIf : remoteElanInterfaces) {
 626                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 627                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 628                 if (elanIfMac == null) {
 629                     continue;
 630                 }
 631                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 632                 if (remoteMacEntries != null) {
 633                     for (MacEntry macEntry : remoteMacEntries) {
 634                         String macAddress = macEntry.getMacAddress().getValue();
 635                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 636                                 dstDpId, elanInstance.getElanInstanceName());
 637                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 638                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 639                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 640                     }
 641                 }
 642             }
 643         }
 644     }
 645
 646     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 647             InterfaceInfo interfaceInfo, ElanInstance elanInstance) throws ElanException {
 648         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 649         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 650         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 651
 652         String interfaceName = elanInterface.getName();
 653         String elanInstanceName = elanInterface.getElanInstanceName();
 654
 655         Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 656         WriteTransaction tx = broker.newWriteOnlyTransaction();
 657         if (elanInfo == null) {
 658             List<String> elanInterfaces = new ArrayList<>();
 659             elanInterfaces.add(interfaceName);
 660             ElanUtils.updateOperationalDataStore(broker, idManager,
 661                     elanInstance, elanInterfaces, tx);
 662         } else {
 663             createElanStateList(elanInstanceName, interfaceName, tx);
 664         }
 665         boolean isFirstInterfaceInDpn = false;
 666         // Specific actions to the DPN where the ElanInterface has been added,
 667         // for example, programming the
 668         // External tunnel table if needed or adding the ElanInterface to the
 669         // DpnInterfaces in the operational DS.
 670         BigInteger dpId = interfaceInfo.getDpId();
 671         DpnInterfaces dpnInterfaces = null;
 672         if (dpId != null && !dpId.equals(ElanConstants.INVALID_DPN)) {
 673             InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 674                     .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
 675             Optional<DpnInterfaces> existingElanDpnInterfaces = ElanUtils.read(broker,
 676                     LogicalDatastoreType.OPERATIONAL, elanDpnInterfaces);
 677             if (!existingElanDpnInterfaces.isPresent()) {
 678                 isFirstInterfaceInDpn = true;
 679                 // ELAN's 1st ElanInterface added to this DPN
 680                 dpnInterfaces = createElanInterfacesList(elanInstanceName, interfaceName, dpId, tx);
 681                 // The 1st ElanInterface in a DPN must program the Ext Tunnel
 682                 // table, but only if Elan has VNI
 683                 if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 684                     setExternalTunnelTable(dpId, elanInstance);
 685                 }
 686                 elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 687             } else {
 688                 List<String> elanInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 689                 elanInterfaces.add(interfaceName);
 690                 if (elanInterfaces.size() == 1) { // 1st dpn interface
 691                     elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 692                 }
 693                 dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, dpId, elanInterfaces, tx);
 694             }
 695         }
 696
 697         // add code to install Local/Remote BC group, unknow DMAC entry,
 698         // terminating service table flow entry
 699         // call bindservice of interfacemanager to create ingress table flow
 700         // enty.
 701         // Add interface to the ElanInterfaceForwardingEntires Container
 702         createElanInterfaceTablesList(interfaceName, tx);
 703         List<ListenableFuture<Void>> futures = new ArrayList<>();
 704         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 705         installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, dpnInterfaces,
 706                 isFirstInterfaceInDpn, null);
 707
 708         // add the vlan provider interface to remote BC group for the elan
 709         // for internal vlan networks
 710         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 711             if (interfaceManager.isExternalInterface(interfaceName)) {
 712                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 713                 handleExternalInterfaceEvent(elanInstance, dpnInterfaces, dpId);
 714             }
 715         }
 716
 717         if (isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 718             //update the remote-DPNs remoteBC group entry with Tunnels
 719             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance, dpId);
 720             setElanAndEtreeBCGrouponOtherDpns(elanInstance, dpId);
 721         }
 722
 723         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 724         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 725                 elanInterface, interfaceInfo, elanInstance, isFirstInterfaceInDpn, this);
 726         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 727         return futures;
 728     }
 729
 730     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 731             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn)
 732             throws ElanException {
 733         String elanInstanceName = elanInstance.getElanInstanceName();
 734         String interfaceName = elanInterface.getName();
 735         WriteTransaction tx = broker.newWriteOnlyTransaction();
 736         BigInteger dpId = interfaceInfo.getDpId();
 737         WriteTransaction writeFlowGroupTx = broker.newWriteOnlyTransaction();
 738         installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 739                 isFirstInterfaceInDpn, tx, writeFlowGroupTx);
 740
 741         List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 742         List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 743
 744         boolean isInterfaceOperational = isOperational(interfaceInfo);
 745         if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 746             for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 747                 InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 748                         staticMacEntry.getMacAddress());
 749                 Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 750                         LogicalDatastoreType.OPERATIONAL, macId);
 751                 if (existingMacEntry.isPresent()) {
 752                     elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 753                             elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 754                             existingMacEntry.get(), tx);
 755                 } else {
 756                     elanForwardingEntriesHandler
 757                             .addElanInterfaceForwardingTableList(elanInstance, interfaceName, staticMacEntry, tx);
 758                 }
 759
 760                 if (isInterfaceOperational) {
 761                     // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 762                     // DPNs
 763                     String macAddress = staticMacEntry.getMacAddress().getValue();
 764                     LOG.info("programming smac and dmacs for {} on source and other DPNs for elan {} and interface {}",
 765                             macAddress, elanInstanceName, interfaceName);
 766                     elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 767                             staticMacEntry.getMacAddress().getValue(), true, writeFlowGroupTx);
 768                 }
 769             }
 770
 771             if (isInterfaceOperational) {
 772                 // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 773                 // on purpose.
 774                 for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 775                     staticMacAddresses.add(staticMacEntry.getMacAddress());
 776                 }
 777                 elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 778                         staticMacAddresses);
 779             }
 780         }
 781         List<ListenableFuture<Void>> futures = new ArrayList<>();
 782         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 783         futures.add(ElanUtils.waitForTransactionToComplete(writeFlowGroupTx));
 784         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 785             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 786             try {
 787                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 788                 if (neutronPort != null) {
 789                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 790                 }
 791             } catch (IllegalArgumentException ex) {
 792                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 793             }
 794         }
 795         return futures;
 796     }
 797
 798     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 799             PhysAddress physAddress) {
 800         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 801         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 802         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 803                 LogicalDatastoreType.OPERATIONAL, macId);
 804
 805         if (!existingMacEntry.isPresent()) {
 806             return;
 807         }
 808
 809         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 810                 .setKey(new MacEntryKey(physAddress)).build();
 811         WriteTransaction tx = broker.newWriteOnlyTransaction();
 812         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 813                 ElanUtils.getElanInstanceByName(broker, elanInstanceName), interfaceInfo, macEntry, tx);
 814         ElanUtils.waitForTransactionToComplete(tx);
 815     }
 816
 817     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 818         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 819                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 820     }
 821
 822     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 823             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, WriteTransaction tx,
 824             WriteTransaction writeFlowGroupTx) throws ElanException {
 825         if (!isOperational(interfaceInfo)) {
 826             return;
 827         }
 828         BigInteger dpId = interfaceInfo.getDpId();
 829         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 830             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, writeFlowGroupTx);
 831         }
 832         setupFilterEqualsTable(elanInstance, interfaceInfo, writeFlowGroupTx);
 833         if (isFirstInterfaceInDpn) {
 834             // Terminating Service , UnknownDMAC Table.
 835             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 836             // for ELAN Instance is VxLAN
 837             if (isVxlan(elanInstance)) {
 838                 setupTerminateServiceTable(elanInstance, dpId, writeFlowGroupTx);
 839             }
 840             setupUnknownDMacTable(elanInstance, dpId, writeFlowGroupTx);
 841             /*
 842              * Install remote DMAC flow. This is required since this DPN is
 843              * added later to the elan instance and remote DMACs of other
 844              * interfaces in this elan instance are not present in the current
 845              * dpn.
 846              */
 847             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 848                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 849                         elanInstance.getElanInstanceName());
 850                 programRemoteDmacFlow(elanInstance, interfaceInfo, writeFlowGroupTx);
 851             }
 852         }
 853         // bind the Elan service to the Interface
 854         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), tx);
 855     }
 856
 857     public void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 858             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, WriteTransaction tx) {
 859         if (!isOperational(interfaceInfo)) {
 860             return;
 861         }
 862         // LocalBroadcast Group creation with elan-Interfaces
 863         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 864         if (isFirstInterfaceInDpn) {
 865             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 866             BigInteger dpId = interfaceInfo.getDpId();
 867             // RemoteBroadcast Group creation
 868             try {
 869                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 870             } catch (InterruptedException e1) {
 871                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 872             }
 873             setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId);
 874             try {
 875                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 876             } catch (InterruptedException e1) {
 877                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 878             }
 879         }
 880     }
 881
 882     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 883             WriteTransaction writeFlowGroupTx) {
 884         int ifTag = interfaceInfo.getInterfaceTag();
 885         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 886                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 887                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 888                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 889                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 890
 891         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flow, writeFlowGroupTx);
 892
 893         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 894                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 895                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 896                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 897
 898         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flowEntry, writeFlowGroupTx);
 899     }
 900
 901     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 902             WriteTransaction deleteFlowGroupTx) {
 903         int ifTag = interfaceInfo.getInterfaceTag();
 904         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 905                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 906                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 907                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 908                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 909
 910         mdsalManager.removeFlowToTx(interfaceInfo.getDpId(), flow, deleteFlowGroupTx);
 911
 912         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 913                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 914                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 915                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 916
 917         mdsalManager.removeFlowToTx(interfaceInfo.getDpId(), flowEntity, deleteFlowGroupTx);
 918     }
 919
 920     private List<Bucket> getRemoteBCGroupBucketInfos(ElanInstance elanInfo, int bucketKeyStart,
 921                                                      InterfaceInfo interfaceInfo, long elanTag) {
 922         return getRemoteBCGroupBuckets(elanInfo, null, interfaceInfo.getDpId(), bucketKeyStart, elanTag);
 923     }
 924
 925     @Nonnull
 926     public List<Bucket> getRemoteBCGroupBuckets(ElanInstance elanInfo, DpnInterfaces dpnInterfaces, BigInteger dpnId,
 927                                                 int bucketId, long elanTag) {
 928         List<Bucket> listBucketInfo = new ArrayList<>();
 929         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
 930
 931         if (isVxlan(elanInfo)) {
 932             listBucketInfo.addAll(getRemoteBCGroupTunnelBuckets(elanDpns, dpnId, bucketId,
 933                     elanUtils.isOpenstackVniSemanticsEnforced() ? elanInfo.getSegmentationId() : elanTag));
 934         }
 935         listBucketInfo.addAll(getRemoteBCGroupExternalPortBuckets(elanDpns, dpnInterfaces, dpnId,
 936             getNextAvailableBucketId(listBucketInfo.size())));
 937         listBucketInfo.addAll(getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnId,
 938             getNextAvailableBucketId(listBucketInfo.size())));
 939         listBucketInfo.addAll(getRemoteBCGroupBucketsOfElanExternalTeps(elanInfo, dpnId,
 940                 getNextAvailableBucketId(listBucketInfo.size())));
 941         return listBucketInfo;
 942     }
 943
 944     private int getNextAvailableBucketId(int bucketSize) {
 945         return bucketSize + 1;
 946     }
 947
 948     @SuppressWarnings("checkstyle:IllegalCatch")
 949     private List<Bucket> getRemoteBCGroupTunnelBuckets(ElanDpnInterfacesList elanDpns, BigInteger dpnId, int bucketId,
 950             long elanTagOrVni) {
 951         List<Bucket> listBucketInfo = new ArrayList<>();
 952         if (elanDpns != null) {
 953             for (DpnInterfaces dpnInterface : elanDpns.getDpnInterfaces()) {
 954                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpnId)
 955                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
 956                     try {
 957                         List<Action> listActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(dpnId,
 958                                 dpnInterface.getDpId(), elanTagOrVni);
 959                         if (listActionInfo.isEmpty()) {
 960                             continue;
 961                         }
 962                         listBucketInfo.add(MDSALUtil.buildBucket(listActionInfo, MDSALUtil.GROUP_WEIGHT, bucketId,
 963                                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
 964                         bucketId++;
 965                     } catch (Exception ex) {
 966                         LOG.error("Logical Group Interface not found between source Dpn - {}, destination Dpn - {} ",
 967                                 dpnId, dpnInterface.getDpId(), ex);
 968                     }
 969                 }
 970             }
 971         }
 972         return listBucketInfo;
 973     }
 974
 975     private List<Bucket> getRemoteBCGroupExternalPortBuckets(ElanDpnInterfacesList elanDpns,
 976             DpnInterfaces dpnInterfaces, BigInteger dpnId, int bucketId) {
 977         DpnInterfaces currDpnInterfaces = dpnInterfaces != null ? dpnInterfaces : getDpnInterfaces(elanDpns, dpnId);
 978         if (currDpnInterfaces == null || !elanUtils.isDpnPresent(currDpnInterfaces.getDpId())
 979                 || currDpnInterfaces.getInterfaces() == null || currDpnInterfaces.getInterfaces().isEmpty()) {
 980             return Collections.emptyList();
 981         }
 982         List<Bucket> listBucketInfo = new ArrayList<>();
 983         for (String interfaceName : currDpnInterfaces.getInterfaces()) {
 984             if (interfaceManager.isExternalInterface(interfaceName)) {
 985                 List<Action> listActionInfo = elanItmUtils.getExternalPortItmEgressAction(interfaceName);
 986                 if (!listActionInfo.isEmpty()) {
 987                     listBucketInfo.add(MDSALUtil.buildBucket(listActionInfo, MDSALUtil.GROUP_WEIGHT, bucketId,
 988                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
 989                     bucketId++;
 990                 }
 991             }
 992         }
 993         return listBucketInfo;
 994     }
 995
 996     private DpnInterfaces getDpnInterfaces(ElanDpnInterfacesList elanDpns, BigInteger dpnId) {
 997         if (elanDpns != null) {
 998             for (DpnInterfaces dpnInterface : elanDpns.getDpnInterfaces()) {
 999                 if (dpnInterface.getDpId().equals(dpnId)) {
1000                     return dpnInterface;
1001                 }
1002             }
1003         }
1004         return null;
1005     }
1006
1007     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId) {
1008         int elanTag = elanInfo.getElanTag().intValue();
1009         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
1010         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId);
1011         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1012         if (etreeInstance != null) {
1013             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
1014             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
1015             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId);
1016         }
1017     }
1018
1019     @SuppressWarnings("checkstyle:IllegalCatch")
1020     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId) {
1021         int bucketId = 0;
1022         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
1023         if (elanDpns != null) {
1024             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1025             for (DpnInterfaces dpnInterface : dpnInterfaces) {
1026                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
1027                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
1028                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
1029                     List<Action> listAction = new ArrayList<>();
1030                     int actionKey = 0;
1031                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1032                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
1033                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1034                     bucketId++;
1035                     for (DpnInterfaces otherFes : dpnInterfaces) {
1036                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
1037                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
1038                             && !otherFes.getInterfaces().isEmpty()) {
1039                             try {
1040                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
1041                                         dpnInterface.getDpId(), otherFes.getDpId(),
1042                                         elanUtils.isOpenstackVniSemanticsEnforced() ? elanInfo.getSegmentationId()
1043                                                 : elanTag);
1044                                 if (!remoteListActionInfo.isEmpty()) {
1045                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
1046                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1047                                     bucketId++;
1048                                 }
1049                             } catch (Exception ex) {
1050                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1051                                         + "Logical Group Interface not found between source Dpn - {}, "
1052                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1053                                 return;
1054                             }
1055                         }
1056                     }
1057                     List<Bucket> elanL2GwDevicesBuckets = getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpId,
1058                             bucketId);
1059                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1060
1061                     if (remoteListBucketInfo.isEmpty()) {
1062                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1063                         continue;
1064                     }
1065                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1066                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1067                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1068                     mdsalManager.syncInstallGroup(dpnInterface.getDpId(), group);
1069                 }
1070             }
1071             try {
1072                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1073             } catch (InterruptedException e1) {
1074                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1075             }
1076         }
1077     }
1078
1079     /**
1080      * Returns the bucket info with the given interface as the only bucket.
1081      */
1082     private Bucket getLocalBCGroupBucketInfo(InterfaceInfo interfaceInfo, int bucketIdStart) {
1083         return MDSALUtil.buildBucket(getInterfacePortActions(interfaceInfo), MDSALUtil.GROUP_WEIGHT, bucketIdStart,
1084                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
1085     }
1086
1087     private List<MatchInfo> buildMatchesForVni(Long vni) {
1088         List<MatchInfo> mkMatches = new ArrayList<>();
1089         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1090         mkMatches.add(match);
1091         return mkMatches;
1092     }
1093
1094     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1095         List<InstructionInfo> mkInstructions = new ArrayList<>();
1096         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1097         return mkInstructions;
1098     }
1099
1100     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1101         List<MatchInfo> mkMatches = new ArrayList<>();
1102         // Matching metadata
1103         mkMatches.add(new MatchMetadata(
1104                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1105         return mkMatches;
1106     }
1107
1108     /**
1109      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1110      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1111      *
1112      * @param elanTag
1113      *            elanTag to be written in metadata when flow is selected
1114      * @return the instructions ready to be installed in a flow
1115      */
1116     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1117         List<InstructionInfo> mkInstructions = new ArrayList<>();
1118         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1119                 .getElanMetadataMask()));
1120         /* applicable for EXTERNAL_TUNNEL_TABLE only
1121         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1122         */
1123         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1124         return mkInstructions;
1125     }
1126
1127     // Install DMAC entry on dst DPN
1128     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1129             BigInteger dstDpId) throws ElanException {
1130         String interfaceName = interfaceInfo.getInterfaceName();
1131         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1132         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1133             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1134             WriteTransaction writeFlowTx = broker.newWriteOnlyTransaction();
1135             for (MacEntry macEntry : macEntries) {
1136                 String macAddress = macEntry.getMacAddress().getValue();
1137                 LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress, interfaceName);
1138                 synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1139                         interfaceInfo.getDpId())) {
1140                     LOG.info("Acquired lock for mac : " + macAddress + ". Proceeding with remote dmac"
1141                             + " install operation.");
1142                     elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress,
1143                             writeFlowTx);
1144                 }
1145             }
1146             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(writeFlowTx));
1147         }
1148         return Collections.emptyList();
1149     }
1150
1151     public void setupElanBroadcastGroups(ElanInstance elanInfo, BigInteger dpnId) {
1152         setupElanBroadcastGroups(elanInfo, null, dpnId);
1153     }
1154
1155     public void setupElanBroadcastGroups(ElanInstance elanInfo, DpnInterfaces dpnInterfaces, BigInteger dpnId) {
1156         setupStandardElanBroadcastGroups(elanInfo, dpnInterfaces, dpnId);
1157         setupLeavesEtreeBroadcastGroups(elanInfo, dpnInterfaces, dpnId);
1158     }
1159
1160     public void setupStandardElanBroadcastGroups(ElanInstance elanInfo, DpnInterfaces dpnInterfaces, BigInteger dpnId) {
1161         List<Bucket> listBucket = new ArrayList<>();
1162         int bucketId = 0;
1163         int actionKey = 0;
1164         Long elanTag = elanInfo.getElanTag();
1165         List<Action> listAction = new ArrayList<>();
1166         listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1167         listBucket.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
1168                 MDSALUtil.WATCH_GROUP));
1169         bucketId++;
1170         List<Bucket> listBucketInfoRemote = getRemoteBCGroupBuckets(elanInfo, dpnInterfaces, dpnId, bucketId, elanTag);
1171         listBucket.addAll(listBucketInfoRemote);
1172         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
1173         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1174                 MDSALUtil.buildBucketLists(listBucket));
1175         LOG.trace("Installing the remote BroadCast Group:{}", group);
1176         mdsalManager.syncInstallGroup(dpnId, group);
1177     }
1178
1179     public void setupLeavesEtreeBroadcastGroups(ElanInstance elanInfo, DpnInterfaces dpnInterfaces, BigInteger dpnId) {
1180         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1181         if (etreeInstance != null) {
1182             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1183             List<Bucket> listBucket = new ArrayList<>();
1184             int bucketId = 0;
1185             int actionKey = 0;
1186             List<Action> listAction = new ArrayList<>();
1187             listAction.add(new ActionGroup(ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag)).buildAction(++actionKey));
1188             listBucket.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
1189                     MDSALUtil.WATCH_GROUP));
1190             bucketId++;
1191             List<Bucket> listBucketInfoRemote = getRemoteBCGroupBuckets(elanInfo, dpnInterfaces, dpnId, bucketId,
1192                     etreeLeafTag);
1193             listBucket.addAll(listBucketInfoRemote);
1194             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
1195             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1196                     MDSALUtil.buildBucketLists(listBucket));
1197             LOG.trace("Installing the remote BroadCast Group:{}", group);
1198             mdsalManager.syncInstallGroup(dpnId, group);
1199         }
1200     }
1201
1202     private void createDropBucket(List<Bucket> listBucket) {
1203         List<Action> actionsInfos = new ArrayList<>();
1204         actionsInfos.add(new ActionDrop().buildAction());
1205         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1206                 MDSALUtil.WATCH_GROUP);
1207         listBucket.add(dropBucket);
1208     }
1209
1210     public void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1211             InterfaceInfo interfaceInfo) {
1212         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1213         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1214     }
1215
1216     public void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1217             InterfaceInfo interfaceInfo) {
1218         List<Bucket> listBucket = new ArrayList<>();
1219         int bucketId = 0;
1220         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1221
1222         List<String> interfaces = new ArrayList<>();
1223         if (newDpnInterface != null) {
1224             interfaces = newDpnInterface.getInterfaces();
1225         }
1226         for (String ifName : interfaces) {
1227             // In case if there is a InterfacePort in the cache which is not in
1228             // operational state, skip processing it
1229             InterfaceInfo ifInfo = interfaceManager
1230                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1231             if (!isOperational(ifInfo)) {
1232                 continue;
1233             }
1234
1235             if (!interfaceManager.isExternalInterface(ifName)) {
1236                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1237                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1238                 bucketId++;
1239             }
1240         }
1241
1242         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1243                 MDSALUtil.buildBucketLists(listBucket));
1244         LOG.trace("installing the localBroadCast Group:{}", group);
1245         mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group);
1246     }
1247
1248     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1249             InterfaceInfo interfaceInfo) {
1250         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1251         if (etreeInstance != null) {
1252             List<Bucket> listBucket = new ArrayList<>();
1253             int bucketId = 0;
1254
1255             List<String> interfaces = new ArrayList<>();
1256             if (newDpnInterface != null) {
1257                 interfaces = newDpnInterface.getInterfaces();
1258             }
1259             for (String ifName : interfaces) {
1260                 // In case if there is a InterfacePort in the cache which is not
1261                 // in
1262                 // operational state, skip processing it
1263                 InterfaceInfo ifInfo = interfaceManager
1264                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1265                 if (!isOperational(ifInfo)) {
1266                     continue;
1267                 }
1268
1269                 if (!interfaceManager.isExternalInterface(ifName)) {
1270                     // only add root interfaces
1271                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1272                 }
1273             }
1274
1275             if (listBucket.isEmpty()) { // No Buckets
1276                 createDropBucket(listBucket);
1277             }
1278
1279             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1280             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1281             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1282                     MDSALUtil.buildBucketLists(listBucket));
1283             LOG.trace("installing the localBroadCast Group:{}", group);
1284             mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group);
1285         }
1286     }
1287
1288     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1289             InterfaceInfo ifInfo) {
1290         EtreeInterface etreeInterface = ElanUtils.getEtreeInterfaceByElanInterfaceName(broker, ifName);
1291         if (etreeInterface != null && etreeInterface.getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1292             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1293                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1294             bucketId++;
1295         }
1296         return bucketId;
1297     }
1298
1299     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1300             WriteTransaction deleteFlowGroupTx) {
1301         BigInteger dpnId = interfaceInfo.getDpId();
1302         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1303         List<Bucket> listBuckets = new ArrayList<>();
1304         int bucketId = 0;
1305         listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
1306         // listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, 1,
1307         // interfaceInfo));
1308         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1309                 MDSALUtil.buildBucketLists(listBuckets));
1310         LOG.trace("deleted the localBroadCast Group:{}", group);
1311         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1312     }
1313
1314     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1315             WriteTransaction deleteFlowGroupTx) {
1316         int bucketId = 0;
1317         int actionKey = 0;
1318         Long elanTag = elanInfo.getElanTag();
1319         List<Bucket> listBuckets = new ArrayList<>();
1320         List<Action> listAction = new ArrayList<>();
1321         listAction.add(new ActionGroup(++actionKey, ElanUtils.getElanLocalBCGId(elanTag)).buildAction());
1322         listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
1323                 MDSALUtil.WATCH_GROUP));
1324         bucketId++;
1325         listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, elanTag));
1326         BigInteger dpnId = interfaceInfo.getDpId();
1327         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1328         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1329                 MDSALUtil.buildBucketLists(listBuckets));
1330         LOG.trace("deleting the remoteBroadCast group:{}", group);
1331         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1332     }
1333
1334     /**
1335      * Installs a flow in the External Tunnel table consisting in translating
1336      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1337      * elanTag that will be used later in the ELANs pipeline.
1338      *
1339      * @param dpnId
1340      *            the dpn id
1341      * @param elanInfo
1342      *            the elan info
1343      */
1344     public void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1345         long elanTag = elanInfo.getElanTag();
1346         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1347                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1348                 elanInfo.getElanInstanceName(), // flowName
1349                 0, // idleTimeout
1350                 0, // hardTimeout
1351                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1352                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1353                 getInstructionsIntOrExtTunnelTable(elanTag));
1354
1355         mdsalManager.installFlow(flowEntity);
1356     }
1357
1358     /**
1359      * Removes, from External Tunnel table, the flow that translates from VNI to
1360      * elanTag. Important: ensure this method is only called whenever there is
1361      * no other ElanInterface in the specified DPN
1362      *
1363      * @param dpnId
1364      *            DPN whose Ext Tunnel table is going to be modified
1365      * @param elanInfo
1366      *            holds the elanTag needed for selecting the flow to be removed
1367      */
1368     public void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1369         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1370         // last ElanInstance plus
1371         // adding a new one does (almost at the same time) are executed in that
1372         // exact order
1373
1374         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1375         FlowEntity flowEntity = new FlowEntityBuilder()
1376             .setDpnId(dpnId)
1377             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1378             .setFlowId(flowId)
1379             .build();
1380         mdsalManager.removeFlow(flowEntity);
1381     }
1382
1383     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1384         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1385         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1386     }
1387
1388     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1389             WriteTransaction writeFlowGroupTx) {
1390         List<? extends MatchInfoBase> listMatchInfoBase;
1391         List<InstructionInfo> instructionInfos;
1392         long serviceId;
1393         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1394             serviceId = elanTag;
1395             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1396             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1397         } else {
1398             serviceId = elanInfo.getSegmentationId();
1399             listMatchInfoBase = buildMatchesForVni(serviceId);
1400             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1401         }
1402         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1403                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1404                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1405                 instructionInfos);
1406         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1407     }
1408
1409     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1410             WriteTransaction writeFlowGroupTx) {
1411         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1412         if (etreeInstance != null) {
1413             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1414         }
1415     }
1416
1417     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1418         long elanTag = elanInfo.getElanTag();
1419         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1420         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1421         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1422     }
1423
1424     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1425             WriteTransaction writeFlowGroupTx) {
1426         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1427         if (etreeLeafTag != null) {
1428             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1429             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1430             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1431         }
1432     }
1433
1434     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1435             WriteTransaction writeFlowGroupTx) {
1436         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1437                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1438                 5, elanInfo.getElanInstanceName(), 0, 0,
1439                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1440                 getMatchesForElanTag(elanTag, /* SH flag */false),
1441                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1442
1443         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1444     }
1445
1446     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1447             WriteTransaction writeFlowGroupTx) {
1448         // only if ELAN can connect to external network, perform the following
1449
1450         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1451             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1452                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1453                     5, elanInfo.getElanInstanceName(), 0, 0,
1454                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1455                     getMatchesForElanTag(elanTag, /* SH flag */true),
1456                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1457             mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1458         }
1459     }
1460
1461
1462     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo, WriteTransaction deleteFlowGroupTx,
1463             long elanTag) {
1464         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1465                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1466         mdsalManager.removeFlowToTx(dpId, flow, deleteFlowGroupTx);
1467
1468         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1469             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1470                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1471                     .build();
1472             mdsalManager.removeFlowToTx(dpId, flow2, deleteFlowGroupTx);
1473         }
1474     }
1475
1476     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1477         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1478     }
1479
1480     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag, WriteTransaction tx) {
1481         if (isStandardElanService(elanInterface)) {
1482             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1483                     elanInterface.getName(), lportTag, tx);
1484         } else { // Etree service
1485             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1486         }
1487     }
1488
1489     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1490             WriteTransaction tx) {
1491         int instructionKey = 0;
1492         List<Instruction> instructions = new ArrayList<>();
1493         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1494                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1495
1496         List<Action> actions = new ArrayList<>();
1497         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1498                 lportTag).buildAction());
1499         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1500                 elanTag).buildAction());
1501         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1502
1503         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1504                 ++instructionKey));
1505
1506         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1507         BoundServices serviceInfo = ElanUtils.getBoundServices(
1508                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1509                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1510         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1511         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1512                 bindServiceId);
1513         if (!existingElanService.isPresent()) {
1514             tx.put(LogicalDatastoreType.CONFIGURATION, bindServiceId, serviceInfo,
1515                     WriteTransaction.CREATE_MISSING_PARENTS);
1516         }
1517     }
1518
1519     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1520             WriteTransaction tx) {
1521         if (elanInterface.getAugmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1522             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1523                     lportTag, tx);
1524         } else {
1525             EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1526             if (etreeInstance == null) {
1527                 LOG.error("EtreeInterface " + elanInterface.getName() + " is associated with a non EtreeInstance: "
1528                         + elanInfo.getElanInstanceName());
1529             } else {
1530                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1531                         elanInterface.getName(), lportTag, tx);
1532             }
1533         }
1534     }
1535
1536     private boolean isStandardElanService(ElanInterface elanInterface) {
1537         return elanInterface.getAugmentation(EtreeInterface.class) == null;
1538     }
1539
1540     protected void unbindService(String interfaceName, WriteTransaction tx) {
1541         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1542         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1543         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1544                 bindServiceId);
1545         if (existingElanService.isPresent()) {
1546             tx.delete(LogicalDatastoreType.CONFIGURATION, bindServiceId);
1547         }
1548     }
1549
1550     private String getFlowRef(long tableId, long elanTag) {
1551         return String.valueOf(tableId) + elanTag;
1552     }
1553
1554     private String getFlowRef(long tableId, long elanTag, String flowName) {
1555         return new StringBuffer().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1556                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1557     }
1558
1559     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1560         return String.valueOf(tableId) + elanTag + shFlag;
1561     }
1562
1563     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1564         List<Action> listAction = new ArrayList<>();
1565         int actionKey = 0;
1566         listAction.add(
1567             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1568         actionKey++;
1569         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1570         return listAction;
1571     }
1572
1573     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1574             List<String> interfaceNames, WriteTransaction tx) {
1575         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1576                 .setKey(new DpnInterfacesKey(dpId)).build();
1577         tx.put(LogicalDatastoreType.OPERATIONAL,
1578                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1579                 WriteTransaction.CREATE_MISSING_PARENTS);
1580         return dpnInterface;
1581     }
1582
1583     /**
1584      * Delete elan dpn interface from operational DS.
1585      *
1586      * @param elanInstanceName
1587      *            the elan instance name
1588      * @param dpId
1589      *            the dp id
1590      */
1591     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId, WriteTransaction tx) {
1592         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1593                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1594         Optional<DpnInterfaces> dpnInterfaces = ElanUtils.read(broker,
1595                 LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1596         if (dpnInterfaces.isPresent()) {
1597             tx.delete(LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1598         }
1599     }
1600
1601     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1602             WriteTransaction tx) {
1603         List<String> interfaceNames = new ArrayList<>();
1604         interfaceNames.add(interfaceName);
1605         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1606                 .setKey(new DpnInterfacesKey(dpId)).build();
1607         tx.put(LogicalDatastoreType.OPERATIONAL,
1608                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1609                 WriteTransaction.CREATE_MISSING_PARENTS);
1610         return dpnInterface;
1611     }
1612
1613     private void createElanInterfaceTablesList(String interfaceName, WriteTransaction tx) {
1614         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1615                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1616         Optional<ElanInterfaceMac> interfaceMacTables = ElanUtils.read(broker,
1617                 LogicalDatastoreType.OPERATIONAL, elanInterfaceMacTables);
1618         // Adding new Elan Interface Port to the operational DataStore without
1619         // Static-Mac Entries..
1620         if (!interfaceMacTables.isPresent()) {
1621             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1622                     .setKey(new ElanInterfaceMacKey(interfaceName)).build();
1623             tx.put(LogicalDatastoreType.OPERATIONAL,
1624                     ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1625                     WriteTransaction.CREATE_MISSING_PARENTS);
1626         }
1627     }
1628
1629     private void createElanStateList(String elanInstanceName, String interfaceName, WriteTransaction tx) {
1630         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1631         Optional<Elan> elanInterfaceLists = ElanUtils.read(broker,
1632                 LogicalDatastoreType.OPERATIONAL, elanInstance);
1633         // Adding new Elan Interface Port to the operational DataStore without
1634         // Static-Mac Entries..
1635         if (elanInterfaceLists.isPresent()) {
1636             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1637             if (interfaceLists == null) {
1638                 interfaceLists = new ArrayList<>();
1639             }
1640             interfaceLists.add(interfaceName);
1641             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1642                     .setKey(new ElanKey(elanInstanceName)).build();
1643             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName),
1644                     elanState, WriteTransaction.CREATE_MISSING_PARENTS);
1645         }
1646     }
1647
1648     private boolean isOperational(InterfaceInfo interfaceInfo) {
1649         if (interfaceInfo == null) {
1650             return false;
1651         }
1652         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1653     }
1654
1655     @SuppressWarnings("checkstyle:IllegalCatch")
1656     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) throws ElanException {
1657         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1658         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1659                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1660         if (dpnInterfaceLists == null) {
1661             return;
1662         }
1663         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1664         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1665             int cnt = 0;
1666             String elanName = elanDpns.getElanInstanceName();
1667             ElanInstance elanInfo = ElanUtils.getElanInstanceByName(broker, elanName);
1668             if (elanInfo == null) {
1669                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1670                         + "skipping this ELAN for tunnel handling", elanName);
1671                 continue;
1672             }
1673             if (ElanUtils.isFlat(elanInfo) || ElanUtils.isVlan(elanInfo)) {
1674                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1675                 continue;
1676             }
1677             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1678             if (dpnInterfaces == null) {
1679                 continue;
1680             }
1681             DpnInterfaces dstDpnIf = null;
1682             for (DpnInterfaces dpnIf : dpnInterfaces) {
1683                 BigInteger dpnIfDpId = dpnIf.getDpId();
1684                 if (dpnIfDpId.equals(srcDpId)) {
1685                     cnt++;
1686                 } else if (dpnIfDpId.equals(dstDpId)) {
1687                     cnt++;
1688                     dstDpnIf = dpnIf;
1689                 }
1690             }
1691             if (cnt == 2) {
1692                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1693                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1694                 jobCoordinator.enqueueJob(elanName, () -> {
1695                     // update Remote BC Group
1696                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1697                     try {
1698                         setupElanBroadcastGroups(elanInfo, srcDpId);
1699                     } catch (RuntimeException e) {
1700                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1701                     }
1702                     Set<String> interfaceLists = new HashSet<>();
1703                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1704                     for (String ifName : interfaceLists) {
1705                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1706                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1707                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1708                             if (isOperational(interfaceInfo)) {
1709                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1710                             }
1711                             return Collections.emptyList();
1712                         }, ElanConstants.JOB_MAX_RETRIES);
1713                     }
1714                     return Collections.emptyList();
1715                 }, ElanConstants.JOB_MAX_RETRIES);
1716             }
1717
1718         }
1719     }
1720
1721     /**
1722      * Handle external tunnel state event.
1723      *
1724      * @param externalTunnel
1725      *            the external tunnel
1726      * @param intrf
1727      *            the interface
1728      * @throws ElanException in case of issues creating the flow objects
1729      */
1730     public void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) throws ElanException {
1731         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1732             return;
1733         }
1734         // dpId/externalNodeId will be available either in source or destination
1735         // based on the tunnel end point
1736         BigInteger dpId = null;
1737         NodeId externalNodeId = null;
1738         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1739             dpId = new BigInteger(externalTunnel.getSourceDevice());
1740             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1741         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1742             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1743             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1744         }
1745         if (dpId == null || externalNodeId == null) {
1746             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1747             return;
1748         }
1749
1750         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1751         if (dpnInterfaceLists == null) {
1752             return;
1753         }
1754         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1755         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1756             String elanName = elanDpns.getElanInstanceName();
1757             ElanInstance elanInfo = ElanUtils.getElanInstanceByName(broker, elanName);
1758
1759             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1760             if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1761                     || dpnInterfaces.getInterfaces().isEmpty()) {
1762                 continue;
1763             }
1764             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1765
1766             setupElanBroadcastGroups(elanInfo, dpId);
1767             // install L2gwDevices local macs in dpn.
1768             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1769             // Install dpn macs on external device
1770             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1771                     externalNodeId);
1772         }
1773         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1774     }
1775
1776     /**
1777      * Installs dpn macs in external device. first it checks if the physical
1778      * locator towards this dpn tep is present or not if the physical locator is
1779      * present go ahead and add the ucast macs otherwise update the mcast mac
1780      * entry to include this dpn tep ip and schedule the job to put ucast macs
1781      * once the physical locator is programmed in device
1782      *
1783      * @param elanName
1784      *            the elan name
1785      * @param lstElanInterfaceNames
1786      *            the lst Elan interface names
1787      * @param dpnId
1788      *            the dpn id
1789      * @param externalNodeId
1790      *            the external node id
1791      */
1792     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1793             NodeId externalNodeId) {
1794         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1795                 externalNodeId.getValue());
1796         if (elanL2GwDevice == null) {
1797             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1798             return;
1799         }
1800         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1801         if (dpnTepIp == null) {
1802             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1803                     externalNodeId);
1804             return;
1805         }
1806
1807         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1808         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1809                 LogicalDatastoreType.OPERATIONAL);
1810         boolean phyLocAlreadyExists =
1811                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1812                 dpnTepIp);
1813         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1814                 phyLocAlreadyExists, String.valueOf(dpnTepIp.getValue()), elanName, externalNodeId.getValue());
1815         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1816
1817         if (phyLocAlreadyExists) {
1818             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1819             return;
1820         }
1821         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1822         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1823     }
1824
1825     /**
1826      * Validate external tunnel state event.
1827      *
1828      * @param externalTunnel
1829      *            the external tunnel
1830      * @param intrf
1831      *            the intrf
1832      * @return true, if successful
1833      */
1834     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1835         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1836             String srcDevice = externalTunnel.getDestinationDevice();
1837             String destDevice = externalTunnel.getSourceDevice();
1838             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1839                     LogicalDatastoreType.CONFIGURATION);
1840             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1841                     otherEndPointExtTunnel);
1842             if (otherEndPointExtTunnel != null) {
1843                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1844                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1845                 if (otherEndPointInterfaceOperational) {
1846                     return true;
1847                 } else {
1848                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1849                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1850                 }
1851             }
1852         }
1853         return false;
1854     }
1855
1856     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1857         List<MatchInfo> mkMatches = new ArrayList<>();
1858         // Matching metadata
1859         mkMatches.add(
1860                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1861         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1862         return mkMatches;
1863     }
1864
1865     public void updateRemoteBroadcastGroupForAllElanDpns(ElanInstance elanInfo) {
1866         List<DpnInterfaces> dpns = elanUtils.getInvolvedDpnsInElan(elanInfo.getElanInstanceName());
1867         for (DpnInterfaces dpn : dpns) {
1868             setupElanBroadcastGroups(elanInfo, dpn.getDpId());
1869         }
1870     }
1871
1872     public List<Bucket> getRemoteBCGroupBucketsOfElanL2GwDevices(ElanInstance elanInfo, BigInteger dpnId,
1873             int bucketId) {
1874         List<Bucket> listBucketInfo = new ArrayList<>();
1875         ConcurrentMap<String, L2GatewayDevice> map = ElanL2GwCacheUtils
1876                 .getInvolvedL2GwDevices(elanInfo.getElanInstanceName());
1877         for (L2GatewayDevice device : map.values()) {
1878             String interfaceName = elanItmUtils.getExternalTunnelInterfaceName(String.valueOf(dpnId),
1879                     device.getHwvtepNodeId());
1880             if (interfaceName == null) {
1881                 continue;
1882             }
1883             List<Action> listActionInfo = elanItmUtils.buildTunnelItmEgressActions(interfaceName,
1884                     ElanUtils.getVxlanSegmentationId(elanInfo));
1885             listBucketInfo.add(MDSALUtil.buildBucket(listActionInfo, MDSALUtil.GROUP_WEIGHT, bucketId,
1886                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1887             bucketId++;
1888         }
1889         return listBucketInfo;
1890     }
1891
1892     public List<Bucket> getRemoteBCGroupBucketsOfElanExternalTeps(ElanInstance elanInfo, BigInteger dpnId,
1893                                                                   int bucketId) {
1894         List<Bucket> listBucketInfo = new ArrayList<>();
1895         List<ExternalTeps> teps = elanInfo.getExternalTeps();
1896         if (teps == null || teps.isEmpty()) {
1897             return listBucketInfo;
1898         }
1899         for (ExternalTeps tep : teps) {
1900             String interfaceName = elanItmUtils.getExternalTunnelInterfaceName(String.valueOf(dpnId),
1901                     tep.getTepIp().toString());
1902             if (interfaceName == null) {
1903                 LOG.error("Could not get interface name to ext tunnel {} {}", dpnId, tep.getTepIp());
1904                 continue;
1905             }
1906             List<Action> listActionInfo = elanItmUtils.buildTunnelItmEgressActions(interfaceName,
1907                     elanInfo.getSegmentationId());
1908             listBucketInfo.add(MDSALUtil.buildBucket(listActionInfo, MDSALUtil.GROUP_WEIGHT, bucketId,
1909                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1910             bucketId++;
1911         }
1912         return listBucketInfo;
1913     }
1914
1915     @Override
1916     protected ElanInterfaceManager getDataTreeChangeListener() {
1917         return this;
1918     }
1919
1920     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1921                                              BigInteger dpId) {
1922         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1923         setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId);
1924         try {
1925             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1926         } catch (InterruptedException e) {
1927             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1928         }
1929     }
1930 }