elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.collect.Lists;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import java.math.BigInteger;
  17 import java.util.ArrayList;
  18 import java.util.Collections;
  19 import java.util.HashSet;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Objects;
  23 import java.util.Queue;
  24 import java.util.Set;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentLinkedQueue;
  27 import javax.annotation.PostConstruct;
  28 import javax.inject.Inject;
  29 import javax.inject.Singleton;
  30 import org.apache.commons.lang3.StringUtils;
  31 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  32 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  33 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  34 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  35 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  36 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  37 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  38 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  39 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  40 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  41 import org.opendaylight.genius.itm.globals.ITMConstants;
  42 import org.opendaylight.genius.mdsalutil.FlowEntity;
  43 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  44 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  45 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  46 import org.opendaylight.genius.mdsalutil.MatchInfo;
  47 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  48 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  49 import org.opendaylight.genius.mdsalutil.NwConstants;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  55 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  56 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  57 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  58 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  59 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  60 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  61 import org.opendaylight.genius.srm.RecoverableListener;
  62 import org.opendaylight.genius.srm.ServiceRecoveryRegistry;
  63 import org.opendaylight.genius.utils.ServiceIndex;
  64 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  65 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  66 import org.opendaylight.netvirt.elan.ElanException;
  67 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  68 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  69 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  70 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  71 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  72 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  73 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  74 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  75 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  76 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  77 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  78 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  79 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  80 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  81 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  82 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  83 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  84 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 125 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 126 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 127 import org.slf4j.Logger;
 128 import org.slf4j.LoggerFactory;
 129
 130 /**
 131  * Class in charge of handling creations, modifications and removals of
 132  * ElanInterfaces.
 133  *
 134  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 135  */
 136 @Singleton
 137 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 138         implements RecoverableListener {
 139     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 140     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 141     private static final boolean SH_FLAG_SET = true;
 142     private static final boolean SH_FLAG_UNSET = false;
 143
 144     private final DataBroker broker;
 145     private final ManagedNewTransactionRunner txRunner;
 146     private final IMdsalApiManager mdsalManager;
 147     private final IInterfaceManager interfaceManager;
 148     private final IdManagerService idManager;
 149     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 150     private final INeutronVpnManager neutronVpnManager;
 151     private final ElanItmUtils elanItmUtils;
 152     private final ElanEtreeUtils elanEtreeUtils;
 153     private final ElanL2GatewayUtils elanL2GatewayUtils;
 154     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 155     private final ElanUtils elanUtils;
 156     private final JobCoordinator jobCoordinator;
 157     private final ElanInstanceCache elanInstanceCache;
 158     private final ElanInterfaceCache elanInterfaceCache;
 159
 160     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 161         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 162
 163     @Inject
 164     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 165                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 166                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 167                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 168                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 169                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 170                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 171                                 final ElanInstanceCache elanInstanceCache,
 172                                 final ElanInterfaceCache elanInterfaceCache,
 173                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 174                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 175         super(ElanInterface.class, ElanInterfaceManager.class);
 176         this.broker = dataBroker;
 177         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 178         this.idManager = managerService;
 179         this.mdsalManager = mdsalApiManager;
 180         this.interfaceManager = interfaceManager;
 181         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 182         this.neutronVpnManager = neutronVpnManager;
 183         this.elanItmUtils = elanItmUtils;
 184         this.elanEtreeUtils = elanEtreeUtils;
 185         this.elanL2GatewayUtils = elanL2GatewayUtils;
 186         this.elanUtils = elanUtils;
 187         this.jobCoordinator = jobCoordinator;
 188         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 189         this.elanInstanceCache = elanInstanceCache;
 190         this.elanInterfaceCache = elanInterfaceCache;
 191         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 192     }
 193
 194     @Override
 195     @PostConstruct
 196     public void init() {
 197         registerListener();
 198     }
 199
 200     @Override
 201     public void registerListener() {
 202         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 203     }
 204
 205     @Override
 206     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 207         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 208     }
 209
 210     @Override
 211     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 212         String interfaceName = del.getName();
 213         ElanInstance elanInfo = elanInstanceCache.get(del.getElanInstanceName()).orNull();
 214         /*
 215          * Handling in case the elan instance is deleted.If the Elan instance is
 216          * deleted, there is no need to explicitly delete the elan interfaces
 217          */
 218         if (elanInfo == null) {
 219             return;
 220         }
 221         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 222         if (interfaceInfo == null && elanInfo.isExternal()) {
 223             // In deleting external network, the underlying ietf Inteface might have been removed
 224             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 225             // from Operational DS instead
 226             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 227         }
 228         String elanInstanceName = elanInfo.getElanInstanceName();
 229         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 230                 interfaceName, interfaceInfo, this);
 231         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 232     }
 233
 234     @SuppressWarnings("checkstyle:ForbiddenMethod")
 235     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 236             InterfaceInfo interfaceInfo) {
 237         String elanName = elanInfo.getElanInstanceName();
 238         boolean isLastElanInterface = false;
 239         boolean isLastInterfaceOnDpn = false;
 240         BigInteger dpId = null;
 241         long elanTag = elanInfo.getElanTag();
 242         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 243         WriteTransaction interfaceTx = broker.newWriteOnlyTransaction();
 244         Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 245         if (elanState == null) {
 246             interfaceTx.cancel();
 247             return Collections.emptyList();
 248         }
 249         WriteTransaction flowTx = broker.newWriteOnlyTransaction();
 250         List<String> elanInterfaces = elanState.getElanInterfaces();
 251         if (elanInterfaces.isEmpty()) {
 252             isLastElanInterface = true;
 253         }
 254         if (interfaceInfo != null) {
 255             dpId = interfaceInfo.getDpId();
 256             DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, dpId,
 257                     interfaceName, elanTag, interfaceTx);
 258             /*
 259              * If there are not elan ports, remove the unknown dmac, terminating
 260              * service table flows, remote/local bc group
 261              */
 262             if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 263                     || dpnInterfaces.getInterfaces().isEmpty()) {
 264                 // No more Elan Interfaces in this DPN
 265                 LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(), dpId);
 266                 if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 267                     removeDefaultTermFlow(dpId, elanInfo.getElanTag());
 268                 }
 269                 removeUnknownDmacFlow(dpId, elanInfo, flowTx, elanInfo.getElanTag());
 270                 removeEtreeUnknownDmacFlow(dpId, elanInfo, flowTx);
 271                 removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 272                 removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 273                 removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 274                 if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 275                     if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 276                         elanUtils.removeTerminatingServiceAction(dpId,
 277                                 elanUtils.getVxlanSegmentationId(elanInfo).intValue());
 278                     }
 279                     unsetExternalTunnelTable(dpId, elanInfo);
 280                 }
 281                 isLastInterfaceOnDpn = true;
 282             } else {
 283                 setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 284             }
 285         }
 286
 287         List<ListenableFuture<Void>> futures = new ArrayList<>();
 288         futures.add(ElanUtils.waitForTransactionToComplete(interfaceTx));
 289         futures.add(ElanUtils.waitForTransactionToComplete(flowTx));
 290
 291         if (isLastInterfaceOnDpn && dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 292             setElanAndEtreeBCGrouponOtherDpns(elanInfo, dpId);
 293         }
 294         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 295                 interfaceName, elanInfo, interfaceInfo, this, isLastElanInterface);
 296         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 297                 ElanConstants.JOB_MAX_RETRIES);
 298
 299         return futures;
 300     }
 301
 302     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 303             WriteTransaction deleteFlowGroupTx) {
 304         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 305         if (etreeLeafTag != null) {
 306             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 307             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 308         }
 309     }
 310
 311     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 312             WriteTransaction deleteFlowGroupTx) {
 313         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 314         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 315     }
 316
 317     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 318             WriteTransaction deleteFlowGroupTx) {
 319         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 320         if (etreeInstance != null) {
 321             BigInteger dpnId = interfaceInfo.getDpId();
 322             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 323             List<Bucket> listBuckets = new ArrayList<>();
 324             int bucketId = 0;
 325             listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
 326             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 327                     MDSALUtil.buildBucketLists(listBuckets));
 328             LOG.trace("deleted the localBroadCast Group:{}", group);
 329             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 330         }
 331     }
 332
 333     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 334             WriteTransaction deleteFlowGroupTx) {
 335         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 336         if (etreeInstance != null) {
 337             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 338             int bucketId = 0;
 339             int actionKey = 0;
 340             List<Bucket> listBuckets = new ArrayList<>();
 341             List<Action> listAction = new ArrayList<>();
 342             listAction.add(new ActionGroup(ElanUtils.getEtreeLeafLocalBCGId(etreeTag)).buildAction(++actionKey));
 343             listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
 344                     MDSALUtil.WATCH_GROUP));
 345             bucketId++;
 346             listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, etreeTag));
 347             BigInteger dpnId = interfaceInfo.getDpId();
 348             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 349             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 350                     MDSALUtil.buildBucketLists(listBuckets));
 351             LOG.trace("deleting the remoteBroadCast group:{}", group);
 352             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 353         }
 354     }
 355
 356     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName, WriteTransaction tx) {
 357         String elanName = elanInfo.getElanInstanceName();
 358         Elan elanState = ElanUtils.getElanByName(broker, elanName);
 359         if (elanState == null) {
 360             return elanState;
 361         }
 362         List<String> elanInterfaces = elanState.getElanInterfaces();
 363         boolean isRemoved = elanInterfaces.remove(interfaceName);
 364         if (!isRemoved) {
 365             return elanState;
 366         }
 367
 368         if (elanInterfaces.isEmpty()) {
 369             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName));
 370             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanMacTableOperationalDataPath(elanName));
 371             tx.delete(LogicalDatastoreType.OPERATIONAL,
 372                     ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 373         } else {
 374             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 375                     .setKey(new ElanKey(elanName)).build();
 376             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName),
 377                     updateElanState);
 378         }
 379         return elanState;
 380     }
 381
 382     private void deleteElanInterfaceFromConfigDS(String interfaceName, WriteTransaction tx) {
 383         // removing the ElanInterface from the config data_store if interface is
 384         // not present in Interface config DS
 385         if (interfaceManager.getInterfaceInfoFromConfigDataStore(interfaceName) == null
 386                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 387             tx.delete(LogicalDatastoreType.CONFIGURATION,
 388                     ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 389         }
 390     }
 391
 392     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 393             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 394         String elanName = elanInfo.getElanInstanceName();
 395         List<ListenableFuture<Void>> futures = new ArrayList<>();
 396         futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(flowTx -> {
 397             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(interfaceTx -> {
 398                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 399                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 400                 Optional<ElanInterfaceMac> existingElanInterfaceMac =
 401                         interfaceTx.read(LogicalDatastoreType.OPERATIONAL, elanInterfaceId).checkedGet();
 402                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 403                 if (interfaceInfo != null) {
 404                     if (existingElanInterfaceMac.isPresent()) {
 405                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 406                         if (existingMacEntries != null) {
 407                             List<PhysAddress> macAddresses = new ArrayList<>();
 408                             for (MacEntry macEntry : existingMacEntries) {
 409                                 PhysAddress macAddress = macEntry.getMacAddress();
 410                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 411                                         macAddress.getValue(), interfaceName);
 412                                 Optional<MacEntry> macEntryOptional =
 413                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 414                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 415                                     interfaceTx.delete(LogicalDatastoreType.OPERATIONAL,
 416                                             ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 417                                 }
 418                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 419                                 macAddresses.add(macAddress);
 420                             }
 421
 422                             // Removing all those MACs from External Devices belonging
 423                             // to this ELAN
 424                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 425                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 426                             }
 427                         }
 428                     }
 429                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 430                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 431                 } else if (existingElanInterfaceMac.isPresent()) {
 432                     // Interface does not exist in ConfigDS, so lets remove everything
 433                     // about that interface related to Elan
 434                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 435                     if (macEntries != null) {
 436                         for (MacEntry macEntry : macEntries) {
 437                             PhysAddress macAddress = macEntry.getMacAddress();
 438                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 439                                 interfaceTx.delete(LogicalDatastoreType.OPERATIONAL,
 440                                         ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 441                             }
 442                         }
 443                     }
 444                 }
 445                 if (existingElanInterfaceMac.isPresent()) {
 446                     interfaceTx.delete(LogicalDatastoreType.OPERATIONAL, elanInterfaceId);
 447                 }
 448                 unbindService(interfaceName, interfaceTx);
 449                 deleteElanInterfaceFromConfigDS(interfaceName, interfaceTx);
 450             }));
 451         }));
 452         return futures;
 453     }
 454
 455     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 456                                                                          String interfaceName, long elanTag,
 457                                                                          WriteTransaction tx) {
 458         DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 459         if (dpnInterfaces != null) {
 460             List<String> interfaceLists = dpnInterfaces.getInterfaces();
 461             if (interfaceLists != null) {
 462                 interfaceLists.remove(interfaceName);
 463             }
 464
 465             if (interfaceLists == null || interfaceLists.isEmpty()) {
 466                 deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 467                 deleteElanDpnInterface(elanName, dpId, tx);
 468             } else {
 469                 dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 470             }
 471         }
 472         return dpnInterfaces;
 473     }
 474
 475     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 476         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 477         for (DpnInterfaces dpnInterface : dpnInterfaces) {
 478             BigInteger currentDpId = dpnInterface.getDpId();
 479             if (!currentDpId.equals(dpId)) {
 480                 for (String elanInterface : dpnInterface.getInterfaces()) {
 481                     ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 482                     if (macs == null || macs.getMacEntry() == null) {
 483                         continue;
 484                     }
 485                     for (MacEntry mac : macs.getMacEntry()) {
 486                         removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 487                         removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 488                     }
 489                 }
 490             }
 491         }
 492     }
 493
 494     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 495         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 496         if (etreeLeafTag != null) {
 497             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac);
 498         }
 499     }
 500
 501     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 502         mdsalManager
 503                 .removeFlow(dpId,
 504                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 505                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 506                                         mac.getMacAddress().getValue(), elanTag)));
 507     }
 508
 509     /*
 510     * Possible Scenarios for update
 511     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 512         then all {1,2,3,4} should be removed
 513
 514         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 515         then all {1,2,3,4} should be added
 516
 517         c. if orig = {1,2,3,4} updated={2,3,4}
 518         then 1 should be removed
 519
 520         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 521         then we should just add 5
 522
 523         e. if orig = {1,2,3,4} updated={2,3,4,5}
 524         then 1 should be removed , 5 should be added
 525     * */
 526     @SuppressWarnings("checkstyle:ForbiddenMethod")
 527     @Override
 528     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 529         // updating the static-Mac Entries for the existing elanInterface
 530         String elanName = update.getElanInstanceName();
 531         String interfaceName = update.getName();
 532
 533         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 534         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 535         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 536         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 537
 538         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 539                 deletedEntry.getMacAddress()));
 540
 541         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 542         * Hence add the macentries for the same.*/
 543         for (StaticMacEntries staticMacEntry : updatedEntries) {
 544             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 545                     staticMacEntry.getMacAddress());
 546             Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 547                     LogicalDatastoreType.OPERATIONAL, macEntryIdentifier);
 548             WriteTransaction tx = broker.newWriteOnlyTransaction();
 549             if (existingMacEntry.isPresent()) {
 550                 elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 551                         elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 552                         tx);
 553             } else {
 554                 elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 555                         elanName, interfaceName, staticMacEntry, tx);
 556             }
 557             ListenableFutures.addErrorLogging(ElanUtils.waitForTransactionToComplete(tx), LOG,
 558                     "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 559         }
 560     }
 561
 562     @Override
 563     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 564         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 565             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 566             String interfaceName = elanInterfaceAdded.getName();
 567             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 568             if (interfaceInfo == null) {
 569                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 570                 return;
 571             }
 572             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 573
 574             if (elanInstance == null) {
 575                 elanInstance = new ElanInstanceBuilder().setElanInstanceName(elanInstanceName)
 576                         .setDescription(elanInterfaceAdded.getDescription()).build();
 577                 // Add the ElanInstance in the Configuration data-store
 578                 List<String> elanInterfaces = new ArrayList<>();
 579                 elanInterfaces.add(interfaceName);
 580                 elanInstance = ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, tx);
 581             }
 582
 583             Long elanTag = elanInstance.getElanTag();
 584             // If elan tag is not updated, then put the elan interface into
 585             // unprocessed entry map and entry. Let entries
 586             // in this map get processed during ELAN update DCN.
 587             if (elanTag == null) {
 588                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 589                 if (elanInterfaces == null) {
 590                     elanInterfaces = new ConcurrentLinkedQueue<>();
 591                 }
 592                 elanInterfaces.add(elanInterfaceAdded);
 593                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 594                 return;
 595             }
 596             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 597                     interfaceInfo, elanInstance, this);
 598             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 599         }), LOG, "Error procedding added ELAN interface");
 600     }
 601
 602     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) throws ElanException {
 603         List<ListenableFuture<Void>> futures = new ArrayList<>();
 604         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 605         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 606             return futures;
 607         }
 608         for (ElanInterface elanInterface : elanInterfaces) {
 609             String interfaceName = elanInterface.getName();
 610             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 611             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 612         }
 613         return futures;
 614     }
 615
 616     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 617             WriteTransaction writeFlowGroupTx) throws ElanException {
 618         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 619                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 620         List<DpnInterfaces> dpnInterfaceLists = null;
 621         if (elanDpnInterfacesList != null) {
 622             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 623         }
 624         if (dpnInterfaceLists == null) {
 625             dpnInterfaceLists = new ArrayList<>();
 626         }
 627         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 628             BigInteger dstDpId = interfaceInfo.getDpId();
 629             if (dpnInterfaces.getDpId().equals(dstDpId)) {
 630                 continue;
 631             }
 632             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 633             for (String remoteIf : remoteElanInterfaces) {
 634                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 635                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 636                 if (elanIfMac == null || remoteInterface == null) {
 637                     continue;
 638                 }
 639                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 640                 if (remoteMacEntries != null) {
 641                     for (MacEntry macEntry : remoteMacEntries) {
 642                         String macAddress = macEntry.getMacAddress().getValue();
 643                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 644                                 dstDpId, elanInstance.getElanInstanceName());
 645                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 646                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 647                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 648                     }
 649                 }
 650             }
 651         }
 652     }
 653
 654     @SuppressWarnings("checkstyle:ForbiddenMethod")
 655     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 656             InterfaceInfo interfaceInfo, ElanInstance elanInstance) throws ElanException {
 657         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 658         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 659         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 660
 661         String interfaceName = elanInterface.getName();
 662         String elanInstanceName = elanInterface.getElanInstanceName();
 663
 664         Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 665         WriteTransaction tx = broker.newWriteOnlyTransaction();
 666         if (elanInfo == null) {
 667             List<String> elanInterfaces = new ArrayList<>();
 668             elanInterfaces.add(interfaceName);
 669             ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, tx);
 670         } else {
 671             createElanStateList(elanInstanceName, interfaceName, tx);
 672         }
 673         boolean isFirstInterfaceInDpn = false;
 674         // Specific actions to the DPN where the ElanInterface has been added,
 675         // for example, programming the
 676         // External tunnel table if needed or adding the ElanInterface to the
 677         // DpnInterfaces in the operational DS.
 678         BigInteger dpId = interfaceInfo.getDpId();
 679         DpnInterfaces dpnInterfaces = null;
 680         if (dpId != null && !dpId.equals(ElanConstants.INVALID_DPN)) {
 681             InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 682                     .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
 683             Optional<DpnInterfaces> existingElanDpnInterfaces = ElanUtils.read(broker,
 684                     LogicalDatastoreType.OPERATIONAL, elanDpnInterfaces);
 685             if (ElanUtils.isVlan(elanInstance)) {
 686                 isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 687                         elanInstanceName, existingElanDpnInterfaces);
 688             } else {
 689                 isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 690             }
 691             if (isFirstInterfaceInDpn) {
 692                 // ELAN's 1st ElanInterface added to this DPN
 693                 if (!existingElanDpnInterfaces.isPresent()) {
 694                     dpnInterfaces = createElanInterfacesList(elanInstanceName, interfaceName, dpId, tx);
 695                 } else {
 696                     List<String> elanInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 697                     elanInterfaces.add(interfaceName);
 698                     dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, dpId,
 699                             elanInterfaces, tx);
 700                 }
 701                 // The 1st ElanInterface in a DPN must program the Ext Tunnel
 702                 // table, but only if Elan has VNI
 703                 if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 704                     setExternalTunnelTable(dpId, elanInstance);
 705                 }
 706                 elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 707             } else {
 708                 List<String> elanInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 709                 elanInterfaces.add(interfaceName);
 710                 if (elanInterfaces.size() == 1) { // 1st dpn interface
 711                     elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 712                 }
 713                 dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, dpId, elanInterfaces, tx);
 714             }
 715         }
 716
 717         // add code to install Local/Remote BC group, unknow DMAC entry,
 718         // terminating service table flow entry
 719         // call bindservice of interfacemanager to create ingress table flow
 720         // enty.
 721         // Add interface to the ElanInterfaceForwardingEntires Container
 722         createElanInterfaceTablesList(interfaceName, tx);
 723         List<ListenableFuture<Void>> futures = new ArrayList<>();
 724         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 725         installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, dpnInterfaces, isFirstInterfaceInDpn);
 726
 727         // add the vlan provider interface to remote BC group for the elan
 728         // for internal vlan networks
 729         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 730             if (interfaceManager.isExternalInterface(interfaceName)) {
 731                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 732                 handleExternalInterfaceEvent(elanInstance, dpnInterfaces, dpId);
 733             }
 734         }
 735
 736         if (isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 737             //update the remote-DPNs remoteBC group entry with Tunnels
 738             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance, dpId);
 739             setElanAndEtreeBCGrouponOtherDpns(elanInstance, dpId);
 740         }
 741
 742         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 743         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 744                 elanInterface, interfaceInfo, elanInstance, isFirstInterfaceInDpn, this);
 745         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 746         return futures;
 747     }
 748
 749     @SuppressWarnings("checkstyle:ForbiddenMethod")
 750     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 751             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn)
 752             throws ElanException {
 753         String elanInstanceName = elanInstance.getElanInstanceName();
 754         String interfaceName = elanInterface.getName();
 755         WriteTransaction tx = broker.newWriteOnlyTransaction();
 756         BigInteger dpId = interfaceInfo.getDpId();
 757         WriteTransaction writeFlowGroupTx = broker.newWriteOnlyTransaction();
 758         installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 759                 isFirstInterfaceInDpn, tx, writeFlowGroupTx);
 760
 761         List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 762         List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 763
 764         boolean isInterfaceOperational = isOperational(interfaceInfo);
 765         if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 766             for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 767                 InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 768                         staticMacEntry.getMacAddress());
 769                 Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 770                         LogicalDatastoreType.OPERATIONAL, macId);
 771                 if (existingMacEntry.isPresent()) {
 772                     elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 773                             elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 774                             existingMacEntry.get(), tx);
 775                 } else {
 776                     elanForwardingEntriesHandler
 777                             .addElanInterfaceForwardingTableList(elanInstanceName, interfaceName, staticMacEntry, tx);
 778                 }
 779
 780                 if (isInterfaceOperational) {
 781                     // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 782                     // DPNs
 783                     String macAddress = staticMacEntry.getMacAddress().getValue();
 784                     LOG.info("programming smac and dmacs for {} on source and other DPNs for elan {} and interface {}",
 785                             macAddress, elanInstanceName, interfaceName);
 786                     elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 787                             staticMacEntry.getMacAddress().getValue(), true, writeFlowGroupTx);
 788                 }
 789             }
 790
 791             if (isInterfaceOperational) {
 792                 // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 793                 // on purpose.
 794                 for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 795                     staticMacAddresses.add(staticMacEntry.getMacAddress());
 796                 }
 797                 elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 798                         staticMacAddresses);
 799             }
 800         }
 801         List<ListenableFuture<Void>> futures = new ArrayList<>();
 802         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 803         futures.add(ElanUtils.waitForTransactionToComplete(writeFlowGroupTx));
 804         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 805             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 806             try {
 807                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 808                 if (neutronPort != null) {
 809                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 810                 }
 811             } catch (IllegalArgumentException ex) {
 812                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 813             }
 814         }
 815         return futures;
 816     }
 817
 818     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 819             PhysAddress physAddress) {
 820         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 821         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 822         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 823                 LogicalDatastoreType.OPERATIONAL, macId);
 824
 825         if (!existingMacEntry.isPresent()) {
 826             return;
 827         }
 828
 829         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 830                 .setKey(new MacEntryKey(physAddress)).build();
 831         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 832                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 833     }
 834
 835     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 836             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 837         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 838         if (!existingElanDpnInterfaces.isPresent()) {
 839             return true;
 840         }
 841         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 842
 843         if (dpnInterfaces.getInterfaces().size() ==  0 || (dpnInterfaces.getInterfaces().size() == 1
 844                 && dpnInterfaces.getInterfaces().contains(routerPortUuid))) {
 845             return true;
 846         }
 847         return false;
 848     }
 849
 850     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 851         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 852                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 853     }
 854
 855     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 856             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, WriteTransaction tx,
 857             WriteTransaction writeFlowGroupTx) throws ElanException {
 858         if (!isOperational(interfaceInfo)) {
 859             return;
 860         }
 861         BigInteger dpId = interfaceInfo.getDpId();
 862         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 863             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, writeFlowGroupTx);
 864         }
 865         setupFilterEqualsTable(elanInstance, interfaceInfo, writeFlowGroupTx);
 866         if (isFirstInterfaceInDpn) {
 867             // Terminating Service , UnknownDMAC Table.
 868             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 869             // for ELAN Instance is VxLAN
 870             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 871                 setupTerminateServiceTable(elanInstance, dpId, writeFlowGroupTx);
 872             }
 873             setupUnknownDMacTable(elanInstance, dpId, writeFlowGroupTx);
 874             /*
 875              * Install remote DMAC flow. This is required since this DPN is
 876              * added later to the elan instance and remote DMACs of other
 877              * interfaces in this elan instance are not present in the current
 878              * dpn.
 879              */
 880             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 881                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 882                         elanInstance.getElanInstanceName());
 883                 programRemoteDmacFlow(elanInstance, interfaceInfo, writeFlowGroupTx);
 884             }
 885         }
 886         // bind the Elan service to the Interface
 887         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), tx);
 888     }
 889
 890     public void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 891             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn) {
 892         if (!isOperational(interfaceInfo)) {
 893             return;
 894         }
 895         // LocalBroadcast Group creation with elan-Interfaces
 896         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 897         if (isFirstInterfaceInDpn) {
 898             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 899             BigInteger dpId = interfaceInfo.getDpId();
 900             // RemoteBroadcast Group creation
 901             try {
 902                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 903             } catch (InterruptedException e1) {
 904                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 905             }
 906             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId);
 907             try {
 908                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 909             } catch (InterruptedException e1) {
 910                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 911             }
 912         }
 913     }
 914
 915     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 916             WriteTransaction writeFlowGroupTx) {
 917         int ifTag = interfaceInfo.getInterfaceTag();
 918         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 919                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 920                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 921                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 922                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 923
 924         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flow, writeFlowGroupTx);
 925
 926         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 927                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 928                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 929                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 930
 931         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flowEntry, writeFlowGroupTx);
 932     }
 933
 934     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 935             WriteTransaction deleteFlowGroupTx) {
 936         int ifTag = interfaceInfo.getInterfaceTag();
 937         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 938                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 939                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 940                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 941                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 942
 943         mdsalManager.removeFlowToTx(interfaceInfo.getDpId(), flow, deleteFlowGroupTx);
 944
 945         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 946                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 947                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 948                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 949
 950         mdsalManager.removeFlowToTx(interfaceInfo.getDpId(), flowEntity, deleteFlowGroupTx);
 951     }
 952
 953     private List<Bucket> getRemoteBCGroupBucketInfos(ElanInstance elanInfo, int bucketKeyStart,
 954                                                      InterfaceInfo interfaceInfo, long elanTag) {
 955         return elanL2GatewayMulticastUtils.getRemoteBCGroupBuckets(elanInfo, null, interfaceInfo.getDpId(),
 956                 bucketKeyStart, elanTag);
 957     }
 958
 959     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId) {
 960         int elanTag = elanInfo.getElanTag().intValue();
 961         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
 962         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId);
 963         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 964         if (etreeInstance != null) {
 965             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
 966             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
 967             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId);
 968         }
 969     }
 970
 971     @SuppressWarnings("checkstyle:IllegalCatch")
 972     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId) {
 973         int bucketId = 0;
 974         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
 975         if (elanDpns != null) {
 976             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
 977             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 978                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
 979                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
 980                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
 981                     List<Action> listAction = new ArrayList<>();
 982                     int actionKey = 0;
 983                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
 984                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
 985                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
 986                     bucketId++;
 987                     for (DpnInterfaces otherFes : dpnInterfaces) {
 988                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
 989                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
 990                             && !otherFes.getInterfaces().isEmpty()) {
 991                             try {
 992                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
 993                                         dpnInterface.getDpId(), otherFes.getDpId(),
 994                                         elanUtils.isOpenstackVniSemanticsEnforced()
 995                                                 ? elanUtils.getVxlanSegmentationId(elanInfo) : elanTag);
 996                                 if (!remoteListActionInfo.isEmpty()) {
 997                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
 998                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
 999                                     bucketId++;
1000                                 }
1001                             } catch (Exception ex) {
1002                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1003                                         + "Logical Group Interface not found between source Dpn - {}, "
1004                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1005                                 return;
1006                             }
1007                         }
1008                     }
1009                     List<Bucket> elanL2GwDevicesBuckets = elanL2GatewayMulticastUtils
1010                             .getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnInterface.getDpId(), bucketId);
1011                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1012
1013                     if (remoteListBucketInfo.isEmpty()) {
1014                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1015                         continue;
1016                     }
1017                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1018                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1019                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1020                     mdsalManager.syncInstallGroup(dpnInterface.getDpId(), group);
1021                 }
1022             }
1023             try {
1024                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1025             } catch (InterruptedException e1) {
1026                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1027             }
1028         }
1029     }
1030
1031     /**
1032      * Returns the bucket info with the given interface as the only bucket.
1033      */
1034     private Bucket getLocalBCGroupBucketInfo(InterfaceInfo interfaceInfo, int bucketIdStart) {
1035         return MDSALUtil.buildBucket(getInterfacePortActions(interfaceInfo), MDSALUtil.GROUP_WEIGHT, bucketIdStart,
1036                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
1037     }
1038
1039     private List<MatchInfo> buildMatchesForVni(Long vni) {
1040         List<MatchInfo> mkMatches = new ArrayList<>();
1041         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1042         mkMatches.add(match);
1043         return mkMatches;
1044     }
1045
1046     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1047         List<InstructionInfo> mkInstructions = new ArrayList<>();
1048         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1049         return mkInstructions;
1050     }
1051
1052     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1053         List<MatchInfo> mkMatches = new ArrayList<>();
1054         // Matching metadata
1055         mkMatches.add(new MatchMetadata(
1056                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1057         return mkMatches;
1058     }
1059
1060     /**
1061      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1062      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1063      *
1064      * @param elanTag
1065      *            elanTag to be written in metadata when flow is selected
1066      * @return the instructions ready to be installed in a flow
1067      */
1068     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1069         List<InstructionInfo> mkInstructions = new ArrayList<>();
1070         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1071                 .getElanMetadataMask()));
1072         /* applicable for EXTERNAL_TUNNEL_TABLE only
1073         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1074         */
1075         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1076         return mkInstructions;
1077     }
1078
1079     // Install DMAC entry on dst DPN
1080     @SuppressWarnings("checkstyle:ForbiddenMethod")
1081     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1082             BigInteger dstDpId) throws ElanException {
1083         String interfaceName = interfaceInfo.getInterfaceName();
1084         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1085         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1086             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1087             WriteTransaction writeFlowTx = broker.newWriteOnlyTransaction();
1088             for (MacEntry macEntry : macEntries) {
1089                 String macAddress = macEntry.getMacAddress().getValue();
1090                 LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress, interfaceName);
1091                 synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1092                         interfaceInfo.getDpId())) {
1093                     LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation", macAddress);
1094                     elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress,
1095                             writeFlowTx);
1096                 }
1097             }
1098             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(writeFlowTx));
1099         }
1100         return Collections.emptyList();
1101     }
1102
1103     private void createDropBucket(List<Bucket> listBucket) {
1104         List<Action> actionsInfos = new ArrayList<>();
1105         actionsInfos.add(new ActionDrop().buildAction());
1106         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1107                 MDSALUtil.WATCH_GROUP);
1108         listBucket.add(dropBucket);
1109     }
1110
1111     public void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1112             InterfaceInfo interfaceInfo) {
1113         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1114         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1115     }
1116
1117     public void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1118             InterfaceInfo interfaceInfo) {
1119         List<Bucket> listBucket = new ArrayList<>();
1120         int bucketId = 0;
1121         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1122
1123         List<String> interfaces = new ArrayList<>();
1124         if (newDpnInterface != null) {
1125             interfaces = newDpnInterface.getInterfaces();
1126         }
1127         for (String ifName : interfaces) {
1128             // In case if there is a InterfacePort in the cache which is not in
1129             // operational state, skip processing it
1130             InterfaceInfo ifInfo = interfaceManager
1131                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1132             if (!isOperational(ifInfo)) {
1133                 continue;
1134             }
1135
1136             if (!interfaceManager.isExternalInterface(ifName)) {
1137                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1138                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1139                 bucketId++;
1140             }
1141         }
1142
1143         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1144                 MDSALUtil.buildBucketLists(listBucket));
1145         LOG.trace("installing the localBroadCast Group:{}", group);
1146         mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group);
1147     }
1148
1149     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1150             InterfaceInfo interfaceInfo) {
1151         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1152         if (etreeInstance != null) {
1153             List<Bucket> listBucket = new ArrayList<>();
1154             int bucketId = 0;
1155
1156             List<String> interfaces = new ArrayList<>();
1157             if (newDpnInterface != null) {
1158                 interfaces = newDpnInterface.getInterfaces();
1159             }
1160             for (String ifName : interfaces) {
1161                 // In case if there is a InterfacePort in the cache which is not
1162                 // in
1163                 // operational state, skip processing it
1164                 InterfaceInfo ifInfo = interfaceManager
1165                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1166                 if (!isOperational(ifInfo)) {
1167                     continue;
1168                 }
1169
1170                 if (!interfaceManager.isExternalInterface(ifName)) {
1171                     // only add root interfaces
1172                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1173                 }
1174             }
1175
1176             if (listBucket.isEmpty()) { // No Buckets
1177                 createDropBucket(listBucket);
1178             }
1179
1180             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1181             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1182             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1183                     MDSALUtil.buildBucketLists(listBucket));
1184             LOG.trace("installing the localBroadCast Group:{}", group);
1185             mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group);
1186         }
1187     }
1188
1189     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1190             InterfaceInfo ifInfo) {
1191         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1192         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1193             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1194                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1195             bucketId++;
1196         }
1197         return bucketId;
1198     }
1199
1200     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1201             WriteTransaction deleteFlowGroupTx) {
1202         BigInteger dpnId = interfaceInfo.getDpId();
1203         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1204         List<Bucket> listBuckets = new ArrayList<>();
1205         int bucketId = 0;
1206         listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
1207         // listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, 1,
1208         // interfaceInfo));
1209         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1210                 MDSALUtil.buildBucketLists(listBuckets));
1211         LOG.trace("deleted the localBroadCast Group:{}", group);
1212         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1213     }
1214
1215     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1216             WriteTransaction deleteFlowGroupTx) {
1217         int bucketId = 0;
1218         int actionKey = 0;
1219         Long elanTag = elanInfo.getElanTag();
1220         List<Bucket> listBuckets = new ArrayList<>();
1221         List<Action> listAction = new ArrayList<>();
1222         listAction.add(new ActionGroup(++actionKey, ElanUtils.getElanLocalBCGId(elanTag)).buildAction());
1223         listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
1224                 MDSALUtil.WATCH_GROUP));
1225         bucketId++;
1226         listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, elanTag));
1227         BigInteger dpnId = interfaceInfo.getDpId();
1228         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1229         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1230                 MDSALUtil.buildBucketLists(listBuckets));
1231         LOG.trace("deleting the remoteBroadCast group:{}", group);
1232         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1233     }
1234
1235     /**
1236      * Installs a flow in the External Tunnel table consisting in translating
1237      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1238      * elanTag that will be used later in the ELANs pipeline.
1239      *
1240      * @param dpnId
1241      *            the dpn id
1242      * @param elanInfo
1243      *            the elan info
1244      */
1245     public void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1246         long elanTag = elanInfo.getElanTag();
1247         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1248                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1249                 elanInfo.getElanInstanceName(), // flowName
1250                 0, // idleTimeout
1251                 0, // hardTimeout
1252                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1253                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1254                 getInstructionsIntOrExtTunnelTable(elanTag));
1255
1256         mdsalManager.installFlow(flowEntity);
1257     }
1258
1259     /**
1260      * Removes, from External Tunnel table, the flow that translates from VNI to
1261      * elanTag. Important: ensure this method is only called whenever there is
1262      * no other ElanInterface in the specified DPN
1263      *
1264      * @param dpnId
1265      *            DPN whose Ext Tunnel table is going to be modified
1266      * @param elanInfo
1267      *            holds the elanTag needed for selecting the flow to be removed
1268      */
1269     public void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1270         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1271         // last ElanInstance plus
1272         // adding a new one does (almost at the same time) are executed in that
1273         // exact order
1274
1275         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1276         FlowEntity flowEntity = new FlowEntityBuilder()
1277             .setDpnId(dpnId)
1278             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1279             .setFlowId(flowId)
1280             .build();
1281         mdsalManager.removeFlow(flowEntity);
1282     }
1283
1284     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1285         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1286         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1287     }
1288
1289     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1290             WriteTransaction writeFlowGroupTx) {
1291         List<? extends MatchInfoBase> listMatchInfoBase;
1292         List<InstructionInfo> instructionInfos;
1293         long serviceId;
1294         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1295             serviceId = elanTag;
1296             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1297             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1298         } else {
1299             serviceId = elanUtils.getVxlanSegmentationId(elanInfo);
1300             listMatchInfoBase = buildMatchesForVni(serviceId);
1301             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1302         }
1303         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1304                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1305                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1306                 instructionInfos);
1307         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1308     }
1309
1310     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1311             WriteTransaction writeFlowGroupTx) {
1312         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1313         if (etreeInstance != null) {
1314             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1315         }
1316     }
1317
1318     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1319         long elanTag = elanInfo.getElanTag();
1320         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1321         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1322         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1323     }
1324
1325     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1326             WriteTransaction writeFlowGroupTx) {
1327         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1328         if (etreeLeafTag != null) {
1329             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1330             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1331             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1332         }
1333     }
1334
1335     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1336             WriteTransaction writeFlowGroupTx) {
1337         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1338                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1339                 5, elanInfo.getElanInstanceName(), 0, 0,
1340                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1341                 getMatchesForElanTag(elanTag, /* SH flag */false),
1342                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1343
1344         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1345     }
1346
1347     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1348             WriteTransaction writeFlowGroupTx) {
1349         // only if ELAN can connect to external network, perform the following
1350
1351         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1352             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1353                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1354                     5, elanInfo.getElanInstanceName(), 0, 0,
1355                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1356                     getMatchesForElanTag(elanTag, /* SH flag */true),
1357                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1358             mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1359         }
1360     }
1361
1362
1363     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo, WriteTransaction deleteFlowGroupTx,
1364             long elanTag) {
1365         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1366                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1367         mdsalManager.removeFlowToTx(dpId, flow, deleteFlowGroupTx);
1368
1369         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1370             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1371                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1372                     .build();
1373             mdsalManager.removeFlowToTx(dpId, flow2, deleteFlowGroupTx);
1374         }
1375     }
1376
1377     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1378         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1379     }
1380
1381     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag, WriteTransaction tx) {
1382         if (isStandardElanService(elanInterface)) {
1383             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1384                     elanInterface.getName(), lportTag, tx);
1385         } else { // Etree service
1386             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1387         }
1388     }
1389
1390     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1391             WriteTransaction tx) {
1392         int instructionKey = 0;
1393         List<Instruction> instructions = new ArrayList<>();
1394         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1395                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1396
1397         List<Action> actions = new ArrayList<>();
1398         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1399                 lportTag).buildAction());
1400         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1401                 elanTag).buildAction());
1402         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1403
1404         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1405                 ++instructionKey));
1406
1407         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1408         BoundServices serviceInfo = ElanUtils.getBoundServices(
1409                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1410                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1411         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1412         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1413                 bindServiceId);
1414         if (!existingElanService.isPresent()) {
1415             tx.put(LogicalDatastoreType.CONFIGURATION, bindServiceId, serviceInfo,
1416                     WriteTransaction.CREATE_MISSING_PARENTS);
1417         }
1418     }
1419
1420     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1421             WriteTransaction tx) {
1422         if (elanInterface.getAugmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1423             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1424                     lportTag, tx);
1425         } else {
1426             EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1427             if (etreeInstance == null) {
1428                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1429                         elanInterface.getName(), elanInfo.getElanInstanceName());
1430             } else {
1431                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1432                         elanInterface.getName(), lportTag, tx);
1433             }
1434         }
1435     }
1436
1437     private boolean isStandardElanService(ElanInterface elanInterface) {
1438         return elanInterface.getAugmentation(EtreeInterface.class) == null;
1439     }
1440
1441     protected void unbindService(String interfaceName, ReadWriteTransaction tx) throws ReadFailedException {
1442         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1443         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1444         if (tx.read(LogicalDatastoreType.CONFIGURATION, bindServiceId).checkedGet().isPresent()) {
1445             tx.delete(LogicalDatastoreType.CONFIGURATION, bindServiceId);
1446         }
1447     }
1448
1449     private String getFlowRef(long tableId, long elanTag) {
1450         return String.valueOf(tableId) + elanTag;
1451     }
1452
1453     private String getFlowRef(long tableId, long elanTag, String flowName) {
1454         return new StringBuffer().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1455                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1456     }
1457
1458     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1459         return String.valueOf(tableId) + elanTag + shFlag;
1460     }
1461
1462     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1463         List<Action> listAction = new ArrayList<>();
1464         int actionKey = 0;
1465         listAction.add(
1466             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1467         actionKey++;
1468         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1469         return listAction;
1470     }
1471
1472     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1473             List<String> interfaceNames, WriteTransaction tx) {
1474         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1475                 .setKey(new DpnInterfacesKey(dpId)).build();
1476         tx.put(LogicalDatastoreType.OPERATIONAL,
1477                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1478                 WriteTransaction.CREATE_MISSING_PARENTS);
1479         return dpnInterface;
1480     }
1481
1482     /**
1483      * Delete elan dpn interface from operational DS.
1484      *
1485      * @param elanInstanceName
1486      *            the elan instance name
1487      * @param dpId
1488      *            the dp id
1489      */
1490     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId, WriteTransaction tx) {
1491         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1492                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1493         Optional<DpnInterfaces> dpnInterfaces = ElanUtils.read(broker,
1494                 LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1495         if (dpnInterfaces.isPresent()) {
1496             tx.delete(LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1497         }
1498     }
1499
1500     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1501             WriteTransaction tx) {
1502         List<String> interfaceNames = new ArrayList<>();
1503         interfaceNames.add(interfaceName);
1504         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1505                 .setKey(new DpnInterfacesKey(dpId)).build();
1506         tx.put(LogicalDatastoreType.OPERATIONAL,
1507                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1508                 WriteTransaction.CREATE_MISSING_PARENTS);
1509         return dpnInterface;
1510     }
1511
1512     private void createElanInterfaceTablesList(String interfaceName, WriteTransaction tx) {
1513         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1514                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1515         Optional<ElanInterfaceMac> interfaceMacTables = ElanUtils.read(broker,
1516                 LogicalDatastoreType.OPERATIONAL, elanInterfaceMacTables);
1517         // Adding new Elan Interface Port to the operational DataStore without
1518         // Static-Mac Entries..
1519         if (!interfaceMacTables.isPresent()) {
1520             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1521                     .setKey(new ElanInterfaceMacKey(interfaceName)).build();
1522             tx.put(LogicalDatastoreType.OPERATIONAL,
1523                     ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1524                     WriteTransaction.CREATE_MISSING_PARENTS);
1525         }
1526     }
1527
1528     private void createElanStateList(String elanInstanceName, String interfaceName, WriteTransaction tx) {
1529         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1530         Optional<Elan> elanInterfaceLists = ElanUtils.read(broker,
1531                 LogicalDatastoreType.OPERATIONAL, elanInstance);
1532         // Adding new Elan Interface Port to the operational DataStore without
1533         // Static-Mac Entries..
1534         if (elanInterfaceLists.isPresent()) {
1535             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1536             if (interfaceLists == null) {
1537                 interfaceLists = new ArrayList<>();
1538             }
1539             interfaceLists.add(interfaceName);
1540             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1541                     .setKey(new ElanKey(elanInstanceName)).build();
1542             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName),
1543                     elanState, WriteTransaction.CREATE_MISSING_PARENTS);
1544         }
1545     }
1546
1547     private boolean isOperational(InterfaceInfo interfaceInfo) {
1548         if (interfaceInfo == null) {
1549             return false;
1550         }
1551         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1552     }
1553
1554     @SuppressWarnings("checkstyle:IllegalCatch")
1555     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) {
1556         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1557         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1558                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1559         if (dpnInterfaceLists == null) {
1560             return;
1561         }
1562         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1563         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1564             int cnt = 0;
1565             String elanName = elanDpns.getElanInstanceName();
1566             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1567             if (elanInfo == null) {
1568                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1569                         + "skipping this ELAN for tunnel handling", elanName);
1570                 continue;
1571             }
1572             if (ElanUtils.isFlat(elanInfo) || ElanUtils.isVlan(elanInfo)) {
1573                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1574                 continue;
1575             }
1576             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1577             if (dpnInterfaces == null) {
1578                 continue;
1579             }
1580             DpnInterfaces dstDpnIf = null;
1581             for (DpnInterfaces dpnIf : dpnInterfaces) {
1582                 BigInteger dpnIfDpId = dpnIf.getDpId();
1583                 if (dpnIfDpId.equals(srcDpId)) {
1584                     cnt++;
1585                 } else if (dpnIfDpId.equals(dstDpId)) {
1586                     cnt++;
1587                     dstDpnIf = dpnIf;
1588                 }
1589             }
1590             if (cnt == 2) {
1591                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1592                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1593                 jobCoordinator.enqueueJob(elanName, () -> {
1594                     // update Remote BC Group
1595                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1596                     try {
1597                         elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId);
1598                     } catch (RuntimeException e) {
1599                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1600                     }
1601                     Set<String> interfaceLists = new HashSet<>();
1602                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1603                     for (String ifName : interfaceLists) {
1604                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1605                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1606                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1607                             if (isOperational(interfaceInfo)) {
1608                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1609                             }
1610                             return Collections.emptyList();
1611                         }, ElanConstants.JOB_MAX_RETRIES);
1612                     }
1613                     return Collections.emptyList();
1614                 }, ElanConstants.JOB_MAX_RETRIES);
1615             }
1616
1617         }
1618     }
1619
1620     /**
1621      * Handle external tunnel state event.
1622      *
1623      * @param externalTunnel
1624      *            the external tunnel
1625      * @param intrf
1626      *            the interface
1627      * @throws ElanException in case of issues creating the flow objects
1628      */
1629     public void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) throws ElanException {
1630         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1631             return;
1632         }
1633         // dpId/externalNodeId will be available either in source or destination
1634         // based on the tunnel end point
1635         BigInteger dpId = null;
1636         NodeId externalNodeId = null;
1637         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1638             dpId = new BigInteger(externalTunnel.getSourceDevice());
1639             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1640         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1641             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1642             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1643         }
1644         if (dpId == null || externalNodeId == null) {
1645             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1646             return;
1647         }
1648
1649         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1650         if (dpnInterfaceLists == null) {
1651             return;
1652         }
1653         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1654         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1655             String elanName = elanDpns.getElanInstanceName();
1656             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1657
1658             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1659             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1660                     || dpnInterfaces.getInterfaces().isEmpty()) {
1661                 continue;
1662             }
1663             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1664
1665             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpId);
1666             // install L2gwDevices local macs in dpn.
1667             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1668             // Install dpn macs on external device
1669             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1670                     externalNodeId);
1671         }
1672         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1673     }
1674
1675     /**
1676      * Installs dpn macs in external device. first it checks if the physical
1677      * locator towards this dpn tep is present or not if the physical locator is
1678      * present go ahead and add the ucast macs otherwise update the mcast mac
1679      * entry to include this dpn tep ip and schedule the job to put ucast macs
1680      * once the physical locator is programmed in device
1681      *
1682      * @param elanName
1683      *            the elan name
1684      * @param lstElanInterfaceNames
1685      *            the lst Elan interface names
1686      * @param dpnId
1687      *            the dpn id
1688      * @param externalNodeId
1689      *            the external node id
1690      */
1691     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1692             NodeId externalNodeId) {
1693         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1694                 externalNodeId.getValue());
1695         if (elanL2GwDevice == null) {
1696             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1697             return;
1698         }
1699         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1700         if (dpnTepIp == null) {
1701             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1702                     externalNodeId);
1703             return;
1704         }
1705
1706         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1707         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1708                 LogicalDatastoreType.OPERATIONAL);
1709         boolean phyLocAlreadyExists =
1710                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1711                 dpnTepIp);
1712         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1713                 phyLocAlreadyExists, String.valueOf(dpnTepIp.getValue()), elanName, externalNodeId.getValue());
1714         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1715
1716         if (phyLocAlreadyExists) {
1717             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1718             return;
1719         }
1720         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1721         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1722     }
1723
1724     /**
1725      * Validate external tunnel state event.
1726      *
1727      * @param externalTunnel
1728      *            the external tunnel
1729      * @param intrf
1730      *            the intrf
1731      * @return true, if successful
1732      */
1733     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1734         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1735             String srcDevice = externalTunnel.getDestinationDevice();
1736             String destDevice = externalTunnel.getSourceDevice();
1737             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1738                     LogicalDatastoreType.CONFIGURATION);
1739             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1740                     otherEndPointExtTunnel);
1741             if (otherEndPointExtTunnel != null) {
1742                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1743                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1744                 if (otherEndPointInterfaceOperational) {
1745                     return true;
1746                 } else {
1747                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1748                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1749                 }
1750             }
1751         }
1752         return false;
1753     }
1754
1755     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1756         List<MatchInfo> mkMatches = new ArrayList<>();
1757         // Matching metadata
1758         mkMatches.add(
1759                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1760         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1761         return mkMatches;
1762     }
1763
1764     @Override
1765     protected ElanInterfaceManager getDataTreeChangeListener() {
1766         return this;
1767     }
1768
1769     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1770                                              BigInteger dpId) {
1771         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1772         elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId);
1773         try {
1774             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1775         } catch (InterruptedException e) {
1776             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1777         }
1778     }
1779 }