vpnservice/elanmanager/elanmanager-impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlan;
  11 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.base.Preconditions;
  15 import com.google.common.collect.Lists;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import java.math.BigInteger;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.HashSet;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.Queue;
  25 import java.util.Set;
  26 import java.util.concurrent.ConcurrentHashMap;
  27 import java.util.concurrent.ConcurrentLinkedQueue;
  28 import java.util.concurrent.ConcurrentMap;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.Nonnull;
  31 import javax.annotation.PostConstruct;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.apache.commons.lang3.StringUtils;
  35 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  36 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  37 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  38 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  39 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  40 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  41 import org.opendaylight.genius.itm.globals.ITMConstants;
  42 import org.opendaylight.genius.mdsalutil.FlowEntity;
  43 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  44 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  45 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  46 import org.opendaylight.genius.mdsalutil.MatchInfo;
  47 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  48 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  49 import org.opendaylight.genius.mdsalutil.NwConstants;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  55 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  56 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  57 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  58 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  59 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  60 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  61 import org.opendaylight.genius.utils.ServiceIndex;
  62 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  63 import org.opendaylight.netvirt.elan.ElanException;
  64 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  65 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  66 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  67 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  68 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  69 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  70 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  71 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  72 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  73 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  74 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  75 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  76 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  77 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  78 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.elan.instance.ExternalTeps;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 120 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 122 import org.slf4j.Logger;
 123 import org.slf4j.LoggerFactory;
 124
 125 /**
 126  * Class in charge of handling creations, modifications and removals of
 127  * ElanInterfaces.
 128  *
 129  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 130  */
 131 @Singleton
 132 @SuppressWarnings("deprecation")
 133 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager> {
 134
 135     private final DataBroker broker;
 136     private final IMdsalApiManager mdsalManager;
 137     private final IInterfaceManager interfaceManager;
 138     private final IdManagerService idManager;
 139     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 140     private final INeutronVpnManager neutronVpnManager;
 141     private final ElanItmUtils elanItmUtils;
 142     private final ElanEtreeUtils elanEtreeUtils;
 143     private final ElanL2GatewayUtils elanL2GatewayUtils;
 144     private ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 145     private ElanUtils elanUtils;
 146     private final JobCoordinator jobCoordinator;
 147
 148     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 149     private static final boolean SH_FLAG_SET = true;
 150     private static final boolean SH_FLAG_UNSET = false;
 151
 152     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 153         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 154
 155     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 156
 157     @Inject
 158     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 159                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 160                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 161                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 162                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 163                                 final JobCoordinator jobCoordinator) {
 164         super(ElanInterface.class, ElanInterfaceManager.class);
 165         this.broker = dataBroker;
 166         this.idManager = managerService;
 167         this.mdsalManager = mdsalApiManager;
 168         this.interfaceManager = interfaceManager;
 169         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 170         this.neutronVpnManager = neutronVpnManager;
 171         this.elanItmUtils = elanItmUtils;
 172         this.elanEtreeUtils = elanEtreeUtils;
 173         this.elanL2GatewayUtils = elanL2GatewayUtils;
 174         this.jobCoordinator = jobCoordinator;
 175     }
 176
 177     /**
 178      * This method is used instead of "regular" standard constructor dependency injection in, only,
 179      * ElanServiceProvider's constructor to wire things together. It was done like this because of the unholy
 180      * triumvirate of unhealthy love triangle between (at least) ElanUtils, ElanInterfaceManager and
 181      * ElanL2GatewayMulticastUtils. The proper solution to be able to get rid of this would be to split up some of these
 182      * relatively big classes into smaller classes, and then inject more fine grained among them.
 183      */
 184     public void setElanUtils(ElanUtils elanUtils) {
 185         this.elanUtils = elanUtils;
 186         this.elanL2GatewayMulticastUtils = elanUtils.getElanL2GatewayMulticastUtils();
 187         this.elanForwardingEntriesHandler.setElanUtils(elanUtils);
 188     }
 189
 190     @Override
 191     @PostConstruct
 192     public void init() {
 193         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 194     }
 195
 196     @Override
 197     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 198         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 199     }
 200
 201     @Override
 202     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 203         String interfaceName = del.getName();
 204         ElanInstance elanInfo = ElanUtils.getElanInstanceByName(broker, del.getElanInstanceName());
 205         /*
 206          * Handling in case the elan instance is deleted.If the Elan instance is
 207          * deleted, there is no need to explicitly delete the elan interfaces
 208          */
 209         if (elanInfo == null) {
 210             return;
 211         }
 212         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 213         if (interfaceInfo == null && elanInfo.isExternal()) {
 214             // In deleting external network, the underlying ietf Inteface might have been removed
 215             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 216             // from Operational DS instead
 217             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 218         }
 219         String elanInstanceName = elanInfo.getElanInstanceName();
 220         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 221                 interfaceName, interfaceInfo, this);
 222         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 223     }
 224
 225     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 226             InterfaceInfo interfaceInfo) {
 227         String elanName = elanInfo.getElanInstanceName();
 228         boolean isLastElanInterface = false;
 229         boolean isLastInterfaceOnDpn = false;
 230         BigInteger dpId = null;
 231         long elanTag = elanInfo.getElanTag();
 232         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 233         WriteTransaction interfaceTx = broker.newWriteOnlyTransaction();
 234         Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 235         if (elanState == null) {
 236             interfaceTx.cancel();
 237             return Collections.emptyList();
 238         }
 239         WriteTransaction flowTx = broker.newWriteOnlyTransaction();
 240         List<String> elanInterfaces = elanState.getElanInterfaces();
 241         if (elanInterfaces.isEmpty()) {
 242             isLastElanInterface = true;
 243         }
 244         if (interfaceInfo != null) {
 245             dpId = interfaceInfo.getDpId();
 246             DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, dpId,
 247                     interfaceName, elanTag, interfaceTx);
 248             /*
 249              * If there are not elan ports, remove the unknown dmac, terminating
 250              * service table flows, remote/local bc group
 251              */
 252             if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 253                     || dpnInterfaces.getInterfaces().isEmpty()) {
 254                 // No more Elan Interfaces in this DPN
 255                 LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(), dpId);
 256                 if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 257                     removeDefaultTermFlow(dpId, elanInfo.getElanTag());
 258                 }
 259                 removeUnknownDmacFlow(dpId, elanInfo, flowTx, elanInfo.getElanTag());
 260                 removeEtreeUnknownDmacFlow(dpId, elanInfo, flowTx);
 261                 removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 262                 removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 263                 removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 264                 if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 265                     if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 266                         elanUtils.removeTerminatingServiceAction(dpId, elanInfo.getSegmentationId().intValue());
 267                     }
 268                     unsetExternalTunnelTable(dpId, elanInfo);
 269                 }
 270                 isLastInterfaceOnDpn = true;
 271             } else {
 272                 setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 273             }
 274         }
 275
 276         List<ListenableFuture<Void>> futures = new ArrayList<>();
 277         futures.add(ElanUtils.waitForTransactionToComplete(interfaceTx));
 278         futures.add(ElanUtils.waitForTransactionToComplete(flowTx));
 279
 280         if (isLastInterfaceOnDpn && dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 281             setElanAndEtreeBCGrouponOtherDpns(elanInfo, dpId);
 282         }
 283         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 284                 interfaceName, elanInfo, interfaceInfo, this, isLastElanInterface);
 285         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 286                 ElanConstants.JOB_MAX_RETRIES);
 287
 288         return futures;
 289     }
 290
 291     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 292             WriteTransaction deleteFlowGroupTx) {
 293         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 294         if (etreeLeafTag != null) {
 295             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 296             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 297         }
 298     }
 299
 300     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 301             WriteTransaction deleteFlowGroupTx) {
 302         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 303         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 304     }
 305
 306     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 307             WriteTransaction deleteFlowGroupTx) {
 308         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 309         if (etreeInstance != null) {
 310             BigInteger dpnId = interfaceInfo.getDpId();
 311             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 312             List<Bucket> listBuckets = new ArrayList<>();
 313             int bucketId = 0;
 314             listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
 315             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 316                     MDSALUtil.buildBucketLists(listBuckets));
 317             LOG.trace("deleted the localBroadCast Group:{}", group);
 318             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 319         }
 320     }
 321
 322     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 323             WriteTransaction deleteFlowGroupTx) {
 324         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 325         if (etreeInstance != null) {
 326             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 327             int bucketId = 0;
 328             int actionKey = 0;
 329             List<Bucket> listBuckets = new ArrayList<>();
 330             List<Action> listAction = new ArrayList<>();
 331             listAction.add(new ActionGroup(ElanUtils.getEtreeLeafLocalBCGId(etreeTag)).buildAction(++actionKey));
 332             listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
 333                     MDSALUtil.WATCH_GROUP));
 334             bucketId++;
 335             listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, etreeTag));
 336             BigInteger dpnId = interfaceInfo.getDpId();
 337             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 338             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 339                     MDSALUtil.buildBucketLists(listBuckets));
 340             LOG.trace("deleting the remoteBroadCast group:{}", group);
 341             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 342         }
 343     }
 344
 345     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName, WriteTransaction tx) {
 346         String elanName = elanInfo.getElanInstanceName();
 347         Elan elanState = ElanUtils.getElanByName(broker, elanName);
 348         if (elanState == null) {
 349             return elanState;
 350         }
 351         List<String> elanInterfaces = elanState.getElanInterfaces();
 352         boolean isRemoved = elanInterfaces.remove(interfaceName);
 353         if (!isRemoved) {
 354             return elanState;
 355         }
 356
 357         if (elanInterfaces.isEmpty()) {
 358             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName));
 359             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanMacTableOperationalDataPath(elanName));
 360             tx.delete(LogicalDatastoreType.OPERATIONAL,
 361                     ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 362         } else {
 363             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 364                     .setKey(new ElanKey(elanName)).build();
 365             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName),
 366                     updateElanState);
 367         }
 368         return elanState;
 369     }
 370
 371     private void deleteElanInterfaceFromConfigDS(String interfaceName, WriteTransaction tx) {
 372         // removing the ElanInterface from the config data_store if interface is
 373         // not present in Interface config DS
 374         if (interfaceManager.getInterfaceInfoFromConfigDataStore(interfaceName) == null
 375                 && ElanUtils.getElanInterfaceByElanInterfaceName(broker, interfaceName) != null) {
 376             tx.delete(LogicalDatastoreType.CONFIGURATION,
 377                     ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 378         }
 379     }
 380
 381     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 382             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 383         String elanName = elanInfo.getElanInstanceName();
 384         WriteTransaction interfaceTx = broker.newWriteOnlyTransaction();
 385         WriteTransaction flowTx = broker.newWriteOnlyTransaction();
 386         InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 387                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 388         Optional<ElanInterfaceMac> existingElanInterfaceMac = ElanUtils.read(broker,
 389                 LogicalDatastoreType.OPERATIONAL, elanInterfaceId);
 390         LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 391         if (interfaceInfo != null) {
 392             if (existingElanInterfaceMac.isPresent()) {
 393                 List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 394                 List<MacEntry> macEntries = new ArrayList<>();
 395                 if (existingMacEntries != null && !existingMacEntries.isEmpty()) {
 396                     macEntries.addAll(existingMacEntries);
 397                 }
 398                 List<PhysAddress> macAddresses = macEntries.stream().map(macEntry -> {
 399                     PhysAddress macAddress = macEntry.getMacAddress();
 400                     LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 401                             macAddress.getValue(), interfaceName);
 402                     Optional<MacEntry> macEntryOptional = elanUtils.getMacEntryForElanInstance(elanName,
 403                             macAddress);
 404                     if (!isLastElanInterface && macEntryOptional.isPresent()) {
 405                         interfaceTx.delete(LogicalDatastoreType.OPERATIONAL,
 406                                 ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 407                     }
 408                     elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 409                     return macAddress;
 410                 }).collect(Collectors.toList());
 411
 412                 // Removing all those MACs from External Devices belonging
 413                 // to this ELAN
 414                 if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 415                     elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 416                 }
 417             }
 418             removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 419             removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 420         } else if (existingElanInterfaceMac.isPresent()) {
 421             // Interface does not exist in ConfigDS, so lets remove everything
 422             // about that interface related to Elan
 423             List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 424             if (macEntries != null) {
 425                 for (MacEntry macEntry : macEntries) {
 426                     PhysAddress macAddress = macEntry.getMacAddress();
 427                     if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 428                         interfaceTx.delete(LogicalDatastoreType.OPERATIONAL,
 429                                 ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 430                     }
 431                 }
 432             }
 433         }
 434         if (existingElanInterfaceMac.isPresent()) {
 435             interfaceTx.delete(LogicalDatastoreType.OPERATIONAL, elanInterfaceId);
 436         }
 437         unbindService(interfaceName, interfaceTx);
 438         deleteElanInterfaceFromConfigDS(interfaceName, interfaceTx);
 439         List<ListenableFuture<Void>> futures = new ArrayList<>();
 440         futures.add(ElanUtils.waitForTransactionToComplete(interfaceTx));
 441         futures.add(ElanUtils.waitForTransactionToComplete(flowTx));
 442         return futures;
 443     }
 444
 445     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 446                                                                          String interfaceName, long elanTag,
 447                                                                          WriteTransaction tx) {
 448         DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 449         if (dpnInterfaces != null) {
 450             List<String> interfaceLists = dpnInterfaces.getInterfaces();
 451             if (interfaceLists != null) {
 452                 interfaceLists.remove(interfaceName);
 453             }
 454
 455             if (interfaceLists == null || interfaceLists.isEmpty()) {
 456                 deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 457                 deleteElanDpnInterface(elanName, dpId, tx);
 458             } else {
 459                 dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 460             }
 461         }
 462         return dpnInterfaces;
 463     }
 464
 465     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 466         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 467         for (DpnInterfaces dpnInterface : dpnInterfaces) {
 468             BigInteger currentDpId = dpnInterface.getDpId();
 469             if (!currentDpId.equals(dpId)) {
 470                 for (String elanInterface : dpnInterface.getInterfaces()) {
 471                     ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 472                     if (macs == null || macs.getMacEntry() == null) {
 473                         continue;
 474                     }
 475                     for (MacEntry mac : macs.getMacEntry()) {
 476                         removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 477                         removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 478                     }
 479                 }
 480             }
 481         }
 482     }
 483
 484     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 485         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 486         if (etreeLeafTag != null) {
 487             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac);
 488         }
 489     }
 490
 491     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 492         mdsalManager
 493                 .removeFlow(dpId,
 494                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 495                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 496                                         mac.getMacAddress().getValue(), elanTag)));
 497     }
 498
 499     /*
 500     * Possible Scenarios for update
 501     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 502         then all {1,2,3,4} should be removed
 503
 504         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 505         then all {1,2,3,4} should be added
 506
 507         c. if orig = {1,2,3,4} updated={2,3,4}
 508         then 1 should be removed
 509
 510         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 511         then we should just add 5
 512
 513         e. if orig = {1,2,3,4} updated={2,3,4,5}
 514         then 1 should be removed , 5 should be added
 515     * */
 516     @Override
 517     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 518         // updating the static-Mac Entries for the existing elanInterface
 519         String elanName = update.getElanInstanceName();
 520         String interfaceName = update.getName();
 521
 522         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 523         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 524         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 525         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 526
 527         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 528                 deletedEntry.getMacAddress()));
 529
 530         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 531         * Hence add the macentries for the same.*/
 532         for (StaticMacEntries staticMacEntry : updatedEntries) {
 533             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 534                     staticMacEntry.getMacAddress());
 535             Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 536                     LogicalDatastoreType.OPERATIONAL, macEntryIdentifier);
 537             WriteTransaction tx = broker.newWriteOnlyTransaction();
 538             if (existingMacEntry.isPresent()) {
 539                 elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 540                         elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 541                         tx);
 542             } else {
 543                 elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 544                         ElanUtils.getElanInstanceByName(broker, elanName), interfaceName, staticMacEntry, tx);
 545             }
 546             ElanUtils.waitForTransactionToComplete(tx);
 547         }
 548     }
 549
 550     @Override
 551     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 552         String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 553         String interfaceName = elanInterfaceAdded.getName();
 554         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 555         if (interfaceInfo == null) {
 556             LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 557             return;
 558         }
 559         ElanInstance elanInstance = ElanUtils.getElanInstanceByName(broker, elanInstanceName);
 560
 561         if (elanInstance == null) {
 562             elanInstance = new ElanInstanceBuilder().setElanInstanceName(elanInstanceName)
 563                     .setDescription(elanInterfaceAdded.getDescription()).build();
 564             // Add the ElanInstance in the Configuration data-store
 565             WriteTransaction tx = broker.newWriteOnlyTransaction();
 566             List<String> elanInterfaces = new ArrayList<>();
 567             elanInterfaces.add(interfaceName);
 568             ElanUtils.updateOperationalDataStore(broker, idManager,
 569                     elanInstance, elanInterfaces, tx);
 570             ElanUtils.waitForTransactionToComplete(tx);
 571             elanInstance = ElanUtils.getElanInstanceByName(broker, elanInstanceName);
 572         }
 573
 574         Long elanTag = elanInstance.getElanTag();
 575         // If elan tag is not updated, then put the elan interface into
 576         // unprocessed entry map and entry. Let entries
 577         // in this map get processed during ELAN update DCN.
 578         if (elanTag == null) {
 579             ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 580             if (elanInterfaces == null) {
 581                 elanInterfaces = new ConcurrentLinkedQueue<>();
 582             }
 583             elanInterfaces.add(elanInterfaceAdded);
 584             unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 585             return;
 586         }
 587         InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 588                 interfaceInfo, elanInstance, this);
 589         jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 590     }
 591
 592     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) throws ElanException {
 593         List<ListenableFuture<Void>> futures = new ArrayList<>();
 594         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 595         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 596             return futures;
 597         }
 598         for (ElanInterface elanInterface : elanInterfaces) {
 599             String interfaceName = elanInterface.getName();
 600             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 601             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 602         }
 603         return futures;
 604     }
 605
 606     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 607             WriteTransaction writeFlowGroupTx) throws ElanException {
 608         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 609                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 610         List<DpnInterfaces> dpnInterfaceLists = null;
 611         if (elanDpnInterfacesList != null) {
 612             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 613         }
 614         if (dpnInterfaceLists == null) {
 615             dpnInterfaceLists = new ArrayList<>();
 616         }
 617         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 618             BigInteger dstDpId = interfaceInfo.getDpId();
 619             if (dpnInterfaces.getDpId().equals(dstDpId)) {
 620                 continue;
 621             }
 622             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 623             for (String remoteIf : remoteElanInterfaces) {
 624                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 625                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 626                 if (elanIfMac == null) {
 627                     continue;
 628                 }
 629                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 630                 if (remoteMacEntries != null) {
 631                     for (MacEntry macEntry : remoteMacEntries) {
 632                         String macAddress = macEntry.getMacAddress().getValue();
 633                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 634                                 dstDpId, elanInstance.getElanInstanceName());
 635                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 636                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 637                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 638                     }
 639                 }
 640             }
 641         }
 642     }
 643
 644     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 645             InterfaceInfo interfaceInfo, ElanInstance elanInstance) throws ElanException {
 646         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 647         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 648         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 649
 650         String interfaceName = elanInterface.getName();
 651         String elanInstanceName = elanInterface.getElanInstanceName();
 652
 653         Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 654         WriteTransaction tx = broker.newWriteOnlyTransaction();
 655         if (elanInfo == null) {
 656             List<String> elanInterfaces = new ArrayList<>();
 657             elanInterfaces.add(interfaceName);
 658             ElanUtils.updateOperationalDataStore(broker, idManager,
 659                     elanInstance, elanInterfaces, tx);
 660         } else {
 661             createElanStateList(elanInstanceName, interfaceName, tx);
 662         }
 663         boolean isFirstInterfaceInDpn = false;
 664         // Specific actions to the DPN where the ElanInterface has been added,
 665         // for example, programming the
 666         // External tunnel table if needed or adding the ElanInterface to the
 667         // DpnInterfaces in the operational DS.
 668         BigInteger dpId = interfaceInfo.getDpId();
 669         DpnInterfaces dpnInterfaces = null;
 670         if (dpId != null && !dpId.equals(ElanConstants.INVALID_DPN)) {
 671             InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 672                     .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
 673             Optional<DpnInterfaces> existingElanDpnInterfaces = ElanUtils.read(broker,
 674                     LogicalDatastoreType.OPERATIONAL, elanDpnInterfaces);
 675             if (!existingElanDpnInterfaces.isPresent()) {
 676                 isFirstInterfaceInDpn = true;
 677                 // ELAN's 1st ElanInterface added to this DPN
 678                 dpnInterfaces = createElanInterfacesList(elanInstanceName, interfaceName, dpId, tx);
 679                 // The 1st ElanInterface in a DPN must program the Ext Tunnel
 680                 // table, but only if Elan has VNI
 681                 if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 682                     setExternalTunnelTable(dpId, elanInstance);
 683                 }
 684                 elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 685             } else {
 686                 List<String> elanInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 687                 elanInterfaces.add(interfaceName);
 688                 if (elanInterfaces.size() == 1) { // 1st dpn interface
 689                     elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 690                 }
 691                 dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, dpId, elanInterfaces, tx);
 692             }
 693         }
 694
 695         // add code to install Local/Remote BC group, unknow DMAC entry,
 696         // terminating service table flow entry
 697         // call bindservice of interfacemanager to create ingress table flow
 698         // enty.
 699         // Add interface to the ElanInterfaceForwardingEntires Container
 700         createElanInterfaceTablesList(interfaceName, tx);
 701         List<ListenableFuture<Void>> futures = new ArrayList<>();
 702         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 703         installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, dpnInterfaces,
 704                 isFirstInterfaceInDpn, null);
 705
 706         // add the vlan provider interface to remote BC group for the elan
 707         // for internal vlan networks
 708         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 709             if (interfaceManager.isExternalInterface(interfaceName)) {
 710                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 711                 handleExternalInterfaceEvent(elanInstance, dpnInterfaces, dpId);
 712             }
 713         }
 714
 715         if (isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 716             //update the remote-DPNs remoteBC group entry with Tunnels
 717             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance, dpId);
 718             setElanAndEtreeBCGrouponOtherDpns(elanInstance, dpId);
 719         }
 720
 721         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 722         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 723                 elanInterface, interfaceInfo, elanInstance, isFirstInterfaceInDpn, this);
 724         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 725         return futures;
 726     }
 727
 728     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 729             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn)
 730             throws ElanException {
 731         String elanInstanceName = elanInstance.getElanInstanceName();
 732         String interfaceName = elanInterface.getName();
 733         WriteTransaction tx = broker.newWriteOnlyTransaction();
 734         BigInteger dpId = interfaceInfo.getDpId();
 735         WriteTransaction writeFlowGroupTx = broker.newWriteOnlyTransaction();
 736         installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 737                 isFirstInterfaceInDpn, tx, writeFlowGroupTx);
 738
 739         List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 740         List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 741
 742         boolean isInterfaceOperational = isOperational(interfaceInfo);
 743         if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 744             for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 745                 InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 746                         staticMacEntry.getMacAddress());
 747                 Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 748                         LogicalDatastoreType.OPERATIONAL, macId);
 749                 if (existingMacEntry.isPresent()) {
 750                     elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 751                             elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 752                             existingMacEntry.get(), tx);
 753                 } else {
 754                     elanForwardingEntriesHandler
 755                             .addElanInterfaceForwardingTableList(elanInstance, interfaceName, staticMacEntry, tx);
 756                 }
 757
 758                 if (isInterfaceOperational) {
 759                     // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 760                     // DPNs
 761                     String macAddress = staticMacEntry.getMacAddress().getValue();
 762                     LOG.info("programming smac and dmacs for {} on source and other DPNs for elan {} and interface {}",
 763                             macAddress, elanInstanceName, interfaceName);
 764                     elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 765                             staticMacEntry.getMacAddress().getValue(), true, writeFlowGroupTx);
 766                 }
 767             }
 768
 769             if (isInterfaceOperational) {
 770                 // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 771                 // on purpose.
 772                 for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 773                     staticMacAddresses.add(staticMacEntry.getMacAddress());
 774                 }
 775                 elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 776                         staticMacAddresses);
 777             }
 778         }
 779         List<ListenableFuture<Void>> futures = new ArrayList<>();
 780         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 781         futures.add(ElanUtils.waitForTransactionToComplete(writeFlowGroupTx));
 782         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 783             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 784             try {
 785                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 786                 if (neutronPort != null) {
 787                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 788                 }
 789             } catch (IllegalArgumentException ex) {
 790                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 791             }
 792         }
 793         return futures;
 794     }
 795
 796     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 797             PhysAddress physAddress) {
 798         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 799         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 800         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 801                 LogicalDatastoreType.OPERATIONAL, macId);
 802
 803         if (!existingMacEntry.isPresent()) {
 804             return;
 805         }
 806
 807         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 808                 .setKey(new MacEntryKey(physAddress)).build();
 809         WriteTransaction tx = broker.newWriteOnlyTransaction();
 810         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 811                 ElanUtils.getElanInstanceByName(broker, elanInstanceName), interfaceInfo, macEntry, tx);
 812         ElanUtils.waitForTransactionToComplete(tx);
 813     }
 814
 815     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 816         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 817                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 818     }
 819
 820     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 821             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, WriteTransaction tx,
 822             WriteTransaction writeFlowGroupTx) throws ElanException {
 823         if (!isOperational(interfaceInfo)) {
 824             return;
 825         }
 826         BigInteger dpId = interfaceInfo.getDpId();
 827         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 828             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, writeFlowGroupTx);
 829         }
 830         setupFilterEqualsTable(elanInstance, interfaceInfo, writeFlowGroupTx);
 831         if (isFirstInterfaceInDpn) {
 832             // Terminating Service , UnknownDMAC Table.
 833             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 834             // for ELAN Instance is VxLAN
 835             if (isVxlan(elanInstance)) {
 836                 setupTerminateServiceTable(elanInstance, dpId, writeFlowGroupTx);
 837             }
 838             setupUnknownDMacTable(elanInstance, dpId, writeFlowGroupTx);
 839             /*
 840              * Install remote DMAC flow. This is required since this DPN is
 841              * added later to the elan instance and remote DMACs of other
 842              * interfaces in this elan instance are not present in the current
 843              * dpn.
 844              */
 845             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 846                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 847                         elanInstance.getElanInstanceName());
 848                 programRemoteDmacFlow(elanInstance, interfaceInfo, writeFlowGroupTx);
 849             }
 850         }
 851         // bind the Elan service to the Interface
 852         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), tx);
 853     }
 854
 855     public void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 856             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, WriteTransaction tx) {
 857         if (!isOperational(interfaceInfo)) {
 858             return;
 859         }
 860         // LocalBroadcast Group creation with elan-Interfaces
 861         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 862         if (isFirstInterfaceInDpn) {
 863             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 864             BigInteger dpId = interfaceInfo.getDpId();
 865             // RemoteBroadcast Group creation
 866             try {
 867                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 868             } catch (InterruptedException e1) {
 869                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 870             }
 871             setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId);
 872             try {
 873                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 874             } catch (InterruptedException e1) {
 875                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 876             }
 877         }
 878     }
 879
 880     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 881             WriteTransaction writeFlowGroupTx) {
 882         int ifTag = interfaceInfo.getInterfaceTag();
 883         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 884                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 885                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 886                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 887                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 888
 889         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flow, writeFlowGroupTx);
 890
 891         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 892                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 893                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 894                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 895
 896         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flowEntry, writeFlowGroupTx);
 897     }
 898
 899     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 900             WriteTransaction deleteFlowGroupTx) {
 901         int ifTag = interfaceInfo.getInterfaceTag();
 902         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 903                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 904                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 905                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 906                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 907
 908         mdsalManager.removeFlowToTx(interfaceInfo.getDpId(), flow, deleteFlowGroupTx);
 909
 910         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 911                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 912                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 913                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 914
 915         mdsalManager.removeFlowToTx(interfaceInfo.getDpId(), flowEntity, deleteFlowGroupTx);
 916     }
 917
 918     private List<Bucket> getRemoteBCGroupBucketInfos(ElanInstance elanInfo, int bucketKeyStart,
 919                                                      InterfaceInfo interfaceInfo, long elanTag) {
 920         return getRemoteBCGroupBuckets(elanInfo, null, interfaceInfo.getDpId(), bucketKeyStart, elanTag);
 921     }
 922
 923     @Nonnull
 924     public List<Bucket> getRemoteBCGroupBuckets(ElanInstance elanInfo, DpnInterfaces dpnInterfaces, BigInteger dpnId,
 925                                                 int bucketId, long elanTag) {
 926         List<Bucket> listBucketInfo = new ArrayList<>();
 927         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
 928
 929         if (isVxlan(elanInfo)) {
 930             listBucketInfo.addAll(getRemoteBCGroupTunnelBuckets(elanDpns, dpnId, bucketId,
 931                     elanUtils.isOpenstackVniSemanticsEnforced() ? elanInfo.getSegmentationId() : elanTag));
 932         }
 933         listBucketInfo.addAll(getRemoteBCGroupExternalPortBuckets(elanDpns, dpnInterfaces, dpnId,
 934             getNextAvailableBucketId(listBucketInfo.size())));
 935         listBucketInfo.addAll(getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnId,
 936             getNextAvailableBucketId(listBucketInfo.size())));
 937         listBucketInfo.addAll(getRemoteBCGroupBucketsOfElanExternalTeps(elanInfo, dpnId,
 938                 getNextAvailableBucketId(listBucketInfo.size())));
 939         return listBucketInfo;
 940     }
 941
 942     private int getNextAvailableBucketId(int bucketSize) {
 943         return bucketSize + 1;
 944     }
 945
 946     @SuppressWarnings("checkstyle:IllegalCatch")
 947     private List<Bucket> getRemoteBCGroupTunnelBuckets(ElanDpnInterfacesList elanDpns, BigInteger dpnId, int bucketId,
 948             long elanTagOrVni) {
 949         List<Bucket> listBucketInfo = new ArrayList<>();
 950         if (elanDpns != null) {
 951             for (DpnInterfaces dpnInterface : elanDpns.getDpnInterfaces()) {
 952                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpnId)
 953                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
 954                     try {
 955                         List<Action> listActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(dpnId,
 956                                 dpnInterface.getDpId(), elanTagOrVni);
 957                         if (listActionInfo.isEmpty()) {
 958                             continue;
 959                         }
 960                         listBucketInfo.add(MDSALUtil.buildBucket(listActionInfo, MDSALUtil.GROUP_WEIGHT, bucketId,
 961                                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
 962                         bucketId++;
 963                     } catch (Exception ex) {
 964                         LOG.error("Logical Group Interface not found between source Dpn - {}, destination Dpn - {} ",
 965                                 dpnId, dpnInterface.getDpId(), ex);
 966                     }
 967                 }
 968             }
 969         }
 970         return listBucketInfo;
 971     }
 972
 973     private List<Bucket> getRemoteBCGroupExternalPortBuckets(ElanDpnInterfacesList elanDpns,
 974             DpnInterfaces dpnInterfaces, BigInteger dpnId, int bucketId) {
 975         DpnInterfaces currDpnInterfaces = dpnInterfaces != null ? dpnInterfaces : getDpnInterfaces(elanDpns, dpnId);
 976         if (currDpnInterfaces == null || !elanUtils.isDpnPresent(currDpnInterfaces.getDpId())
 977                 || currDpnInterfaces.getInterfaces() == null || currDpnInterfaces.getInterfaces().isEmpty()) {
 978             return Collections.emptyList();
 979         }
 980         List<Bucket> listBucketInfo = new ArrayList<>();
 981         for (String interfaceName : currDpnInterfaces.getInterfaces()) {
 982             if (interfaceManager.isExternalInterface(interfaceName)) {
 983                 List<Action> listActionInfo = elanItmUtils.getExternalPortItmEgressAction(interfaceName);
 984                 if (!listActionInfo.isEmpty()) {
 985                     listBucketInfo.add(MDSALUtil.buildBucket(listActionInfo, MDSALUtil.GROUP_WEIGHT, bucketId,
 986                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
 987                     bucketId++;
 988                 }
 989             }
 990         }
 991         return listBucketInfo;
 992     }
 993
 994     private DpnInterfaces getDpnInterfaces(ElanDpnInterfacesList elanDpns, BigInteger dpnId) {
 995         if (elanDpns != null) {
 996             for (DpnInterfaces dpnInterface : elanDpns.getDpnInterfaces()) {
 997                 if (dpnInterface.getDpId().equals(dpnId)) {
 998                     return dpnInterface;
 999                 }
1000             }
1001         }
1002         return null;
1003     }
1004
1005     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId) {
1006         int elanTag = elanInfo.getElanTag().intValue();
1007         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
1008         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId);
1009         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1010         if (etreeInstance != null) {
1011             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
1012             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
1013             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId);
1014         }
1015     }
1016
1017     @SuppressWarnings("checkstyle:IllegalCatch")
1018     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId) {
1019         int bucketId = 0;
1020         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
1021         if (elanDpns != null) {
1022             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1023             for (DpnInterfaces dpnInterface : dpnInterfaces) {
1024                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
1025                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
1026                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
1027                     List<Action> listAction = new ArrayList<>();
1028                     int actionKey = 0;
1029                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1030                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
1031                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1032                     bucketId++;
1033                     for (DpnInterfaces otherFes : dpnInterfaces) {
1034                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
1035                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
1036                             && !otherFes.getInterfaces().isEmpty()) {
1037                             try {
1038                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
1039                                         dpnInterface.getDpId(), otherFes.getDpId(),
1040                                         elanUtils.isOpenstackVniSemanticsEnforced() ? elanInfo.getSegmentationId()
1041                                                 : elanTag);
1042                                 if (!remoteListActionInfo.isEmpty()) {
1043                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
1044                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1045                                     bucketId++;
1046                                 }
1047                             } catch (Exception ex) {
1048                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1049                                         + "Logical Group Interface not found between source Dpn - {}, "
1050                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1051                                 return;
1052                             }
1053                         }
1054                     }
1055                     List<Bucket> elanL2GwDevicesBuckets = getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpId,
1056                             bucketId);
1057                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1058
1059                     if (remoteListBucketInfo.isEmpty()) {
1060                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1061                         continue;
1062                     }
1063                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1064                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1065                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1066                     mdsalManager.syncInstallGroup(dpnInterface.getDpId(), group);
1067                 }
1068             }
1069             try {
1070                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1071             } catch (InterruptedException e1) {
1072                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1073             }
1074         }
1075     }
1076
1077     /**
1078      * Returns the bucket info with the given interface as the only bucket.
1079      */
1080     private Bucket getLocalBCGroupBucketInfo(InterfaceInfo interfaceInfo, int bucketIdStart) {
1081         return MDSALUtil.buildBucket(getInterfacePortActions(interfaceInfo), MDSALUtil.GROUP_WEIGHT, bucketIdStart,
1082                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
1083     }
1084
1085     private List<MatchInfo> buildMatchesForVni(Long vni) {
1086         List<MatchInfo> mkMatches = new ArrayList<>();
1087         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1088         mkMatches.add(match);
1089         return mkMatches;
1090     }
1091
1092     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1093         List<InstructionInfo> mkInstructions = new ArrayList<>();
1094         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1095         return mkInstructions;
1096     }
1097
1098     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1099         List<MatchInfo> mkMatches = new ArrayList<>();
1100         // Matching metadata
1101         mkMatches.add(new MatchMetadata(
1102                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1103         return mkMatches;
1104     }
1105
1106     /**
1107      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1108      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1109      *
1110      * @param elanTag
1111      *            elanTag to be written in metadata when flow is selected
1112      * @return the instructions ready to be installed in a flow
1113      */
1114     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1115         List<InstructionInfo> mkInstructions = new ArrayList<>();
1116         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1117                 .getElanMetadataMask()));
1118         /* applicable for EXTERNAL_TUNNEL_TABLE only
1119         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1120         */
1121         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1122         return mkInstructions;
1123     }
1124
1125     // Install DMAC entry on dst DPN
1126     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1127             BigInteger dstDpId) throws ElanException {
1128         String interfaceName = interfaceInfo.getInterfaceName();
1129         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1130         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1131             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1132             WriteTransaction writeFlowTx = broker.newWriteOnlyTransaction();
1133             for (MacEntry macEntry : macEntries) {
1134                 String macAddress = macEntry.getMacAddress().getValue();
1135                 LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress, interfaceName);
1136                 synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1137                         interfaceInfo.getDpId())) {
1138                     LOG.info("Acquired lock for mac : " + macAddress + ". Proceeding with remote dmac"
1139                             + " install operation.");
1140                     elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress,
1141                             writeFlowTx);
1142                 }
1143             }
1144             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(writeFlowTx));
1145         }
1146         return Collections.emptyList();
1147     }
1148
1149     public void setupElanBroadcastGroups(ElanInstance elanInfo, BigInteger dpnId) {
1150         setupElanBroadcastGroups(elanInfo, null, dpnId);
1151     }
1152
1153     public void setupElanBroadcastGroups(ElanInstance elanInfo, DpnInterfaces dpnInterfaces, BigInteger dpnId) {
1154         setupStandardElanBroadcastGroups(elanInfo, dpnInterfaces, dpnId);
1155         setupLeavesEtreeBroadcastGroups(elanInfo, dpnInterfaces, dpnId);
1156     }
1157
1158     public void setupStandardElanBroadcastGroups(ElanInstance elanInfo, DpnInterfaces dpnInterfaces, BigInteger dpnId) {
1159         List<Bucket> listBucket = new ArrayList<>();
1160         int bucketId = 0;
1161         int actionKey = 0;
1162         Long elanTag = elanInfo.getElanTag();
1163         List<Action> listAction = new ArrayList<>();
1164         listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1165         listBucket.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
1166                 MDSALUtil.WATCH_GROUP));
1167         bucketId++;
1168         List<Bucket> listBucketInfoRemote = getRemoteBCGroupBuckets(elanInfo, dpnInterfaces, dpnId, bucketId, elanTag);
1169         listBucket.addAll(listBucketInfoRemote);
1170         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
1171         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1172                 MDSALUtil.buildBucketLists(listBucket));
1173         LOG.trace("Installing the remote BroadCast Group:{}", group);
1174         mdsalManager.syncInstallGroup(dpnId, group);
1175     }
1176
1177     public void setupLeavesEtreeBroadcastGroups(ElanInstance elanInfo, DpnInterfaces dpnInterfaces, BigInteger dpnId) {
1178         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1179         if (etreeInstance != null) {
1180             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1181             List<Bucket> listBucket = new ArrayList<>();
1182             int bucketId = 0;
1183             int actionKey = 0;
1184             List<Action> listAction = new ArrayList<>();
1185             listAction.add(new ActionGroup(ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag)).buildAction(++actionKey));
1186             listBucket.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
1187                     MDSALUtil.WATCH_GROUP));
1188             bucketId++;
1189             List<Bucket> listBucketInfoRemote = getRemoteBCGroupBuckets(elanInfo, dpnInterfaces, dpnId, bucketId,
1190                     etreeLeafTag);
1191             listBucket.addAll(listBucketInfoRemote);
1192             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
1193             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1194                     MDSALUtil.buildBucketLists(listBucket));
1195             LOG.trace("Installing the remote BroadCast Group:{}", group);
1196             mdsalManager.syncInstallGroup(dpnId, group);
1197         }
1198     }
1199
1200     private void createDropBucket(List<Bucket> listBucket) {
1201         List<Action> actionsInfos = new ArrayList<>();
1202         actionsInfos.add(new ActionDrop().buildAction());
1203         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1204                 MDSALUtil.WATCH_GROUP);
1205         listBucket.add(dropBucket);
1206     }
1207
1208     public void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1209             InterfaceInfo interfaceInfo) {
1210         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1211         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1212     }
1213
1214     public void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1215             InterfaceInfo interfaceInfo) {
1216         List<Bucket> listBucket = new ArrayList<>();
1217         int bucketId = 0;
1218         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1219
1220         List<String> interfaces = new ArrayList<>();
1221         if (newDpnInterface != null) {
1222             interfaces = newDpnInterface.getInterfaces();
1223         }
1224         for (String ifName : interfaces) {
1225             // In case if there is a InterfacePort in the cache which is not in
1226             // operational state, skip processing it
1227             InterfaceInfo ifInfo = interfaceManager
1228                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1229             if (!isOperational(ifInfo)) {
1230                 continue;
1231             }
1232
1233             if (!interfaceManager.isExternalInterface(ifName)) {
1234                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1235                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1236                 bucketId++;
1237             }
1238         }
1239
1240         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1241                 MDSALUtil.buildBucketLists(listBucket));
1242         LOG.trace("installing the localBroadCast Group:{}", group);
1243         mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group);
1244     }
1245
1246     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1247             InterfaceInfo interfaceInfo) {
1248         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1249         if (etreeInstance != null) {
1250             List<Bucket> listBucket = new ArrayList<>();
1251             int bucketId = 0;
1252
1253             List<String> interfaces = new ArrayList<>();
1254             if (newDpnInterface != null) {
1255                 interfaces = newDpnInterface.getInterfaces();
1256             }
1257             for (String ifName : interfaces) {
1258                 // In case if there is a InterfacePort in the cache which is not
1259                 // in
1260                 // operational state, skip processing it
1261                 InterfaceInfo ifInfo = interfaceManager
1262                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1263                 if (!isOperational(ifInfo)) {
1264                     continue;
1265                 }
1266
1267                 if (!interfaceManager.isExternalInterface(ifName)) {
1268                     // only add root interfaces
1269                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1270                 }
1271             }
1272
1273             if (listBucket.isEmpty()) { // No Buckets
1274                 createDropBucket(listBucket);
1275             }
1276
1277             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1278             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1279             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1280                     MDSALUtil.buildBucketLists(listBucket));
1281             LOG.trace("installing the localBroadCast Group:{}", group);
1282             mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group);
1283         }
1284     }
1285
1286     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1287             InterfaceInfo ifInfo) {
1288         EtreeInterface etreeInterface = ElanUtils.getEtreeInterfaceByElanInterfaceName(broker, ifName);
1289         if (etreeInterface != null && etreeInterface.getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1290             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1291                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1292             bucketId++;
1293         }
1294         return bucketId;
1295     }
1296
1297     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1298             WriteTransaction deleteFlowGroupTx) {
1299         BigInteger dpnId = interfaceInfo.getDpId();
1300         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1301         List<Bucket> listBuckets = new ArrayList<>();
1302         int bucketId = 0;
1303         listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
1304         // listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, 1,
1305         // interfaceInfo));
1306         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1307                 MDSALUtil.buildBucketLists(listBuckets));
1308         LOG.trace("deleted the localBroadCast Group:{}", group);
1309         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1310     }
1311
1312     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1313             WriteTransaction deleteFlowGroupTx) {
1314         int bucketId = 0;
1315         int actionKey = 0;
1316         Long elanTag = elanInfo.getElanTag();
1317         List<Bucket> listBuckets = new ArrayList<>();
1318         List<Action> listAction = new ArrayList<>();
1319         listAction.add(new ActionGroup(++actionKey, ElanUtils.getElanLocalBCGId(elanTag)).buildAction());
1320         listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
1321                 MDSALUtil.WATCH_GROUP));
1322         bucketId++;
1323         listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, elanTag));
1324         BigInteger dpnId = interfaceInfo.getDpId();
1325         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1326         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1327                 MDSALUtil.buildBucketLists(listBuckets));
1328         LOG.trace("deleting the remoteBroadCast group:{}", group);
1329         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1330     }
1331
1332     /**
1333      * Installs a flow in the External Tunnel table consisting in translating
1334      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1335      * elanTag that will be used later in the ELANs pipeline.
1336      *
1337      * @param dpnId
1338      *            the dpn id
1339      * @param elanInfo
1340      *            the elan info
1341      */
1342     public void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1343         long elanTag = elanInfo.getElanTag();
1344         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1345                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1346                 elanInfo.getElanInstanceName(), // flowName
1347                 0, // idleTimeout
1348                 0, // hardTimeout
1349                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1350                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1351                 getInstructionsIntOrExtTunnelTable(elanTag));
1352
1353         mdsalManager.installFlow(flowEntity);
1354     }
1355
1356     /**
1357      * Removes, from External Tunnel table, the flow that translates from VNI to
1358      * elanTag. Important: ensure this method is only called whenever there is
1359      * no other ElanInterface in the specified DPN
1360      *
1361      * @param dpnId
1362      *            DPN whose Ext Tunnel table is going to be modified
1363      * @param elanInfo
1364      *            holds the elanTag needed for selecting the flow to be removed
1365      */
1366     public void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1367         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1368         // last ElanInstance plus
1369         // adding a new one does (almost at the same time) are executed in that
1370         // exact order
1371
1372         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1373         FlowEntity flowEntity = new FlowEntityBuilder()
1374             .setDpnId(dpnId)
1375             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1376             .setFlowId(flowId)
1377             .build();
1378         mdsalManager.removeFlow(flowEntity);
1379     }
1380
1381     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1382         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1383         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1384     }
1385
1386     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1387             WriteTransaction writeFlowGroupTx) {
1388         List<? extends MatchInfoBase> listMatchInfoBase;
1389         List<InstructionInfo> instructionInfos;
1390         long serviceId;
1391         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1392             serviceId = elanTag;
1393             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1394             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1395         } else {
1396             serviceId = elanInfo.getSegmentationId();
1397             listMatchInfoBase = buildMatchesForVni(serviceId);
1398             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1399         }
1400         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1401                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1402                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1403                 instructionInfos);
1404         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1405     }
1406
1407     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1408             WriteTransaction writeFlowGroupTx) {
1409         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1410         if (etreeInstance != null) {
1411             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1412         }
1413     }
1414
1415     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1416         long elanTag = elanInfo.getElanTag();
1417         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1418         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1419         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1420     }
1421
1422     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1423             WriteTransaction writeFlowGroupTx) {
1424         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1425         if (etreeLeafTag != null) {
1426             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1427             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1428             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1429         }
1430     }
1431
1432     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1433             WriteTransaction writeFlowGroupTx) {
1434         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1435                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1436                 5, elanInfo.getElanInstanceName(), 0, 0,
1437                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1438                 getMatchesForElanTag(elanTag, /* SH flag */false),
1439                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1440
1441         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1442     }
1443
1444     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1445             WriteTransaction writeFlowGroupTx) {
1446         // only if ELAN can connect to external network, perform the following
1447
1448         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1449             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1450                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1451                     5, elanInfo.getElanInstanceName(), 0, 0,
1452                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1453                     getMatchesForElanTag(elanTag, /* SH flag */true),
1454                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1455             mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1456         }
1457     }
1458
1459
1460     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo, WriteTransaction deleteFlowGroupTx,
1461             long elanTag) {
1462         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1463                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1464         mdsalManager.removeFlowToTx(dpId, flow, deleteFlowGroupTx);
1465
1466         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1467             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1468                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1469                     .build();
1470             mdsalManager.removeFlowToTx(dpId, flow2, deleteFlowGroupTx);
1471         }
1472     }
1473
1474     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1475         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1476     }
1477
1478     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag, WriteTransaction tx) {
1479         if (isStandardElanService(elanInterface)) {
1480             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1481                     elanInterface.getName(), lportTag, tx);
1482         } else { // Etree service
1483             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1484         }
1485     }
1486
1487     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1488             WriteTransaction tx) {
1489         int instructionKey = 0;
1490         List<Instruction> instructions = new ArrayList<>();
1491         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1492                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1493
1494         List<Action> actions = new ArrayList<>();
1495         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1496                 lportTag).buildAction());
1497         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1498                 elanTag).buildAction());
1499         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1500
1501         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1502                 ++instructionKey));
1503
1504         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1505         BoundServices serviceInfo = ElanUtils.getBoundServices(
1506                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1507                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1508         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1509         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1510                 bindServiceId);
1511         if (!existingElanService.isPresent()) {
1512             tx.put(LogicalDatastoreType.CONFIGURATION, bindServiceId, serviceInfo,
1513                     WriteTransaction.CREATE_MISSING_PARENTS);
1514         }
1515     }
1516
1517     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1518             WriteTransaction tx) {
1519         if (elanInterface.getAugmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1520             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1521                     lportTag, tx);
1522         } else {
1523             EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1524             if (etreeInstance == null) {
1525                 LOG.error("EtreeInterface " + elanInterface.getName() + " is associated with a non EtreeInstance: "
1526                         + elanInfo.getElanInstanceName());
1527             } else {
1528                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1529                         elanInterface.getName(), lportTag, tx);
1530             }
1531         }
1532     }
1533
1534     private boolean isStandardElanService(ElanInterface elanInterface) {
1535         return elanInterface.getAugmentation(EtreeInterface.class) == null;
1536     }
1537
1538     protected void unbindService(String interfaceName, WriteTransaction tx) {
1539         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1540         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1541         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1542                 bindServiceId);
1543         if (existingElanService.isPresent()) {
1544             tx.delete(LogicalDatastoreType.CONFIGURATION, bindServiceId);
1545         }
1546     }
1547
1548     private String getFlowRef(long tableId, long elanTag) {
1549         return String.valueOf(tableId) + elanTag;
1550     }
1551
1552     private String getFlowRef(long tableId, long elanTag, String flowName) {
1553         return new StringBuffer().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1554                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1555     }
1556
1557     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1558         return String.valueOf(tableId) + elanTag + shFlag;
1559     }
1560
1561     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1562         List<Action> listAction = new ArrayList<>();
1563         int actionKey = 0;
1564         listAction.add(
1565             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1566         actionKey++;
1567         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1568         return listAction;
1569     }
1570
1571     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1572             List<String> interfaceNames, WriteTransaction tx) {
1573         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1574                 .setKey(new DpnInterfacesKey(dpId)).build();
1575         tx.put(LogicalDatastoreType.OPERATIONAL,
1576                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1577                 WriteTransaction.CREATE_MISSING_PARENTS);
1578         return dpnInterface;
1579     }
1580
1581     /**
1582      * Delete elan dpn interface from operational DS.
1583      *
1584      * @param elanInstanceName
1585      *            the elan instance name
1586      * @param dpId
1587      *            the dp id
1588      */
1589     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId, WriteTransaction tx) {
1590         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1591                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1592         Optional<DpnInterfaces> dpnInterfaces = ElanUtils.read(broker,
1593                 LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1594         if (dpnInterfaces.isPresent()) {
1595             tx.delete(LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1596         }
1597     }
1598
1599     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1600             WriteTransaction tx) {
1601         List<String> interfaceNames = new ArrayList<>();
1602         interfaceNames.add(interfaceName);
1603         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1604                 .setKey(new DpnInterfacesKey(dpId)).build();
1605         tx.put(LogicalDatastoreType.OPERATIONAL,
1606                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1607                 WriteTransaction.CREATE_MISSING_PARENTS);
1608         return dpnInterface;
1609     }
1610
1611     private void createElanInterfaceTablesList(String interfaceName, WriteTransaction tx) {
1612         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1613                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1614         Optional<ElanInterfaceMac> interfaceMacTables = ElanUtils.read(broker,
1615                 LogicalDatastoreType.OPERATIONAL, elanInterfaceMacTables);
1616         // Adding new Elan Interface Port to the operational DataStore without
1617         // Static-Mac Entries..
1618         if (!interfaceMacTables.isPresent()) {
1619             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1620                     .setKey(new ElanInterfaceMacKey(interfaceName)).build();
1621             tx.put(LogicalDatastoreType.OPERATIONAL,
1622                     ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1623                     WriteTransaction.CREATE_MISSING_PARENTS);
1624         }
1625     }
1626
1627     private void createElanStateList(String elanInstanceName, String interfaceName, WriteTransaction tx) {
1628         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1629         Optional<Elan> elanInterfaceLists = ElanUtils.read(broker,
1630                 LogicalDatastoreType.OPERATIONAL, elanInstance);
1631         // Adding new Elan Interface Port to the operational DataStore without
1632         // Static-Mac Entries..
1633         if (elanInterfaceLists.isPresent()) {
1634             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1635             if (interfaceLists == null) {
1636                 interfaceLists = new ArrayList<>();
1637             }
1638             interfaceLists.add(interfaceName);
1639             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1640                     .setKey(new ElanKey(elanInstanceName)).build();
1641             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName),
1642                     elanState, WriteTransaction.CREATE_MISSING_PARENTS);
1643         }
1644     }
1645
1646     private boolean isOperational(InterfaceInfo interfaceInfo) {
1647         if (interfaceInfo == null) {
1648             return false;
1649         }
1650         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1651     }
1652
1653     @SuppressWarnings("checkstyle:IllegalCatch")
1654     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) throws ElanException {
1655         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1656         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1657                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1658         if (dpnInterfaceLists == null) {
1659             return;
1660         }
1661         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1662         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1663             int cnt = 0;
1664             String elanName = elanDpns.getElanInstanceName();
1665             ElanInstance elanInfo = ElanUtils.getElanInstanceByName(broker, elanName);
1666             if (elanInfo == null) {
1667                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1668                         + "skipping this ELAN for tunnel handling", elanName);
1669                 continue;
1670             }
1671             if (ElanUtils.isFlat(elanInfo) || ElanUtils.isVlan(elanInfo)) {
1672                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1673                 continue;
1674             }
1675             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1676             if (dpnInterfaces == null) {
1677                 continue;
1678             }
1679             DpnInterfaces dstDpnIf = null;
1680             for (DpnInterfaces dpnIf : dpnInterfaces) {
1681                 BigInteger dpnIfDpId = dpnIf.getDpId();
1682                 if (dpnIfDpId.equals(srcDpId)) {
1683                     cnt++;
1684                 } else if (dpnIfDpId.equals(dstDpId)) {
1685                     cnt++;
1686                     dstDpnIf = dpnIf;
1687                 }
1688             }
1689             if (cnt == 2) {
1690                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1691                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1692                 jobCoordinator.enqueueJob(elanName, () -> {
1693                     // update Remote BC Group
1694                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1695                     try {
1696                         setupElanBroadcastGroups(elanInfo, srcDpId);
1697                     } catch (RuntimeException e) {
1698                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1699                     }
1700                     Set<String> interfaceLists = new HashSet<>();
1701                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1702                     for (String ifName : interfaceLists) {
1703                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1704                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1705                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1706                             if (isOperational(interfaceInfo)) {
1707                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1708                             }
1709                             return Collections.emptyList();
1710                         }, ElanConstants.JOB_MAX_RETRIES);
1711                     }
1712                     return Collections.emptyList();
1713                 }, ElanConstants.JOB_MAX_RETRIES);
1714             }
1715
1716         }
1717     }
1718
1719     /**
1720      * Handle external tunnel state event.
1721      *
1722      * @param externalTunnel
1723      *            the external tunnel
1724      * @param intrf
1725      *            the interface
1726      * @throws ElanException in case of issues creating the flow objects
1727      */
1728     public void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) throws ElanException {
1729         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1730             return;
1731         }
1732         // dpId/externalNodeId will be available either in source or destination
1733         // based on the tunnel end point
1734         BigInteger dpId = null;
1735         NodeId externalNodeId = null;
1736         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1737             dpId = new BigInteger(externalTunnel.getSourceDevice());
1738             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1739         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1740             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1741             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1742         }
1743         if (dpId == null || externalNodeId == null) {
1744             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1745             return;
1746         }
1747
1748         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1749         if (dpnInterfaceLists == null) {
1750             return;
1751         }
1752         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1753         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1754             String elanName = elanDpns.getElanInstanceName();
1755             ElanInstance elanInfo = ElanUtils.getElanInstanceByName(broker, elanName);
1756
1757             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1758             if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1759                     || dpnInterfaces.getInterfaces().isEmpty()) {
1760                 continue;
1761             }
1762             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1763
1764             setupElanBroadcastGroups(elanInfo, dpId);
1765             // install L2gwDevices local macs in dpn.
1766             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1767             // Install dpn macs on external device
1768             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1769                     externalNodeId);
1770         }
1771         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1772     }
1773
1774     /**
1775      * Installs dpn macs in external device. first it checks if the physical
1776      * locator towards this dpn tep is present or not if the physical locator is
1777      * present go ahead and add the ucast macs otherwise update the mcast mac
1778      * entry to include this dpn tep ip and schedule the job to put ucast macs
1779      * once the physical locator is programmed in device
1780      *
1781      * @param elanName
1782      *            the elan name
1783      * @param lstElanInterfaceNames
1784      *            the lst Elan interface names
1785      * @param dpnId
1786      *            the dpn id
1787      * @param externalNodeId
1788      *            the external node id
1789      */
1790     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1791             NodeId externalNodeId) {
1792         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1793                 externalNodeId.getValue());
1794         if (elanL2GwDevice == null) {
1795             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1796             return;
1797         }
1798         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1799         if (dpnTepIp == null) {
1800             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1801                     externalNodeId);
1802             return;
1803         }
1804
1805         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1806         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1807                 LogicalDatastoreType.OPERATIONAL);
1808         boolean phyLocAlreadyExists =
1809                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1810                 dpnTepIp);
1811         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1812                 phyLocAlreadyExists, String.valueOf(dpnTepIp.getValue()), elanName, externalNodeId.getValue());
1813         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1814
1815         if (phyLocAlreadyExists) {
1816             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1817             return;
1818         }
1819         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1820         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1821     }
1822
1823     /**
1824      * Validate external tunnel state event.
1825      *
1826      * @param externalTunnel
1827      *            the external tunnel
1828      * @param intrf
1829      *            the intrf
1830      * @return true, if successful
1831      */
1832     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1833         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1834             String srcDevice = externalTunnel.getDestinationDevice();
1835             String destDevice = externalTunnel.getSourceDevice();
1836             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1837                     LogicalDatastoreType.CONFIGURATION);
1838             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1839                     otherEndPointExtTunnel);
1840             if (otherEndPointExtTunnel != null) {
1841                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1842                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1843                 if (otherEndPointInterfaceOperational) {
1844                     return true;
1845                 } else {
1846                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1847                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1848                 }
1849             }
1850         }
1851         return false;
1852     }
1853
1854     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1855         List<MatchInfo> mkMatches = new ArrayList<>();
1856         // Matching metadata
1857         mkMatches.add(
1858                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1859         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1860         return mkMatches;
1861     }
1862
1863     public void updateRemoteBroadcastGroupForAllElanDpns(ElanInstance elanInfo) {
1864         List<DpnInterfaces> dpns = elanUtils.getInvolvedDpnsInElan(elanInfo.getElanInstanceName());
1865         for (DpnInterfaces dpn : dpns) {
1866             setupElanBroadcastGroups(elanInfo, dpn.getDpId());
1867         }
1868     }
1869
1870     public List<Bucket> getRemoteBCGroupBucketsOfElanL2GwDevices(ElanInstance elanInfo, BigInteger dpnId,
1871             int bucketId) {
1872         List<Bucket> listBucketInfo = new ArrayList<>();
1873         ConcurrentMap<String, L2GatewayDevice> map = ElanL2GwCacheUtils
1874                 .getInvolvedL2GwDevices(elanInfo.getElanInstanceName());
1875         for (L2GatewayDevice device : map.values()) {
1876             String interfaceName = elanItmUtils.getExternalTunnelInterfaceName(String.valueOf(dpnId),
1877                     device.getHwvtepNodeId());
1878             if (interfaceName == null) {
1879                 continue;
1880             }
1881             List<Action> listActionInfo = elanItmUtils.buildTunnelItmEgressActions(interfaceName,
1882                     ElanUtils.getVxlanSegmentationId(elanInfo));
1883             listBucketInfo.add(MDSALUtil.buildBucket(listActionInfo, MDSALUtil.GROUP_WEIGHT, bucketId,
1884                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1885             bucketId++;
1886         }
1887         return listBucketInfo;
1888     }
1889
1890     public List<Bucket> getRemoteBCGroupBucketsOfElanExternalTeps(ElanInstance elanInfo, BigInteger dpnId,
1891                                                                   int bucketId) {
1892         List<Bucket> listBucketInfo = new ArrayList<>();
1893         List<ExternalTeps> teps = elanInfo.getExternalTeps();
1894         if (teps == null || teps.isEmpty()) {
1895             return listBucketInfo;
1896         }
1897         for (ExternalTeps tep : teps) {
1898             String interfaceName = elanItmUtils.getExternalTunnelInterfaceName(String.valueOf(dpnId),
1899                     tep.getTepIp().toString());
1900             if (interfaceName == null) {
1901                 LOG.error("Could not get interface name to ext tunnel {} {}", dpnId, tep.getTepIp());
1902                 continue;
1903             }
1904             List<Action> listActionInfo = elanItmUtils.buildTunnelItmEgressActions(interfaceName,
1905                     elanInfo.getSegmentationId());
1906             listBucketInfo.add(MDSALUtil.buildBucket(listActionInfo, MDSALUtil.GROUP_WEIGHT, bucketId,
1907                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1908             bucketId++;
1909         }
1910         return listBucketInfo;
1911     }
1912
1913     @Override
1914     protected ElanInterfaceManager getDataTreeChangeListener() {
1915         return this;
1916     }
1917
1918     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1919                                              BigInteger dpId) {
1920         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1921         setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId);
1922         try {
1923             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1924         } catch (InterruptedException e) {
1925             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1926         }
1927     }
1928 }