vpnservice/elanmanager/elanmanager-impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlan;
  11 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.base.Preconditions;
  15 import com.google.common.collect.Lists;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import java.math.BigInteger;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.HashSet;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.Queue;
  25 import java.util.Set;
  26 import java.util.concurrent.ConcurrentHashMap;
  27 import java.util.concurrent.ConcurrentLinkedQueue;
  28 import java.util.concurrent.ConcurrentMap;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.inject.Inject;
  32 import javax.inject.Singleton;
  33 import org.apache.commons.lang3.StringUtils;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  38 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  39 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  40 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  41 import org.opendaylight.genius.itm.globals.ITMConstants;
  42 import org.opendaylight.genius.mdsalutil.FlowEntity;
  43 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  44 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  45 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  46 import org.opendaylight.genius.mdsalutil.MatchInfo;
  47 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  48 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  49 import org.opendaylight.genius.mdsalutil.NwConstants;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  55 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  56 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  57 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  58 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  59 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  60 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  61 import org.opendaylight.genius.utils.ServiceIndex;
  62 import org.opendaylight.netvirt.elan.ElanException;
  63 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  64 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  65 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  66 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  67 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  68 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  69 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  70 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  71 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  72 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  73 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.elan.instance.ExternalTeps;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 114 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 115 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 116 import org.slf4j.Logger;
 117 import org.slf4j.LoggerFactory;
 118
 119 /**
 120  * Class in charge of handling creations, modifications and removals of
 121  * ElanInterfaces.
 122  *
 123  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 124  */
 125 @Singleton
 126 @SuppressWarnings("deprecation")
 127 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 128         implements AutoCloseable {
 129
 130     private final DataBroker broker;
 131     private final IMdsalApiManager mdsalManager;
 132     private final IInterfaceManager interfaceManager;
 133     private final IdManagerService idManager;
 134     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 135     private final INeutronVpnManager neutronVpnManager;
 136     private ElanL2GatewayUtils elanL2GatewayUtils;
 137     private ElanUtils elanUtils;
 138
 139     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 140     private static final boolean SH_FLAG_SET = true;
 141     private static final boolean SH_FLAG_UNSET = false;
 142
 143     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 144         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 145
 146     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 147
 148     @Inject
 149     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 150                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 151                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 152                                 final INeutronVpnManager neutronVpnManager) {
 153         super(ElanInterface.class, ElanInterfaceManager.class);
 154         this.broker = dataBroker;
 155         this.idManager = managerService;
 156         this.mdsalManager = mdsalApiManager;
 157         this.interfaceManager = interfaceManager;
 158         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 159         this.neutronVpnManager = neutronVpnManager;
 160     }
 161
 162     /**
 163      * This method is used instead of "regular" standard constructor dependency injection in, only,
 164      * ElanServiceProvider's constructor to wire things together. It was done like this because of the unholy
 165      * triumvirate of unhealthy love triangle between (at least) ElanUtils, ElanInterfaceManager and
 166      * ElanL2GatewayMulticastUtils. The proper solution to be able to get rid of this would be to split up some of these
 167      * relatively big classes into smaller classes, and then inject more fine grained among them.
 168      */
 169     public void setElanUtils(ElanUtils elanUtils) {
 170         this.elanUtils = elanUtils;
 171         this.elanL2GatewayUtils = elanUtils.getElanL2GatewayUtils();
 172         this.elanForwardingEntriesHandler.setElanUtils(elanUtils);
 173     }
 174
 175     @Override
 176     @PostConstruct
 177     public void init() {
 178         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 179     }
 180
 181     @Override
 182     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 183         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 184     }
 185
 186     @Override
 187     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 188         String interfaceName = del.getName();
 189         ElanInstance elanInfo = ElanUtils.getElanInstanceByName(broker, del.getElanInstanceName());
 190         /*
 191          * Handling in case the elan instance is deleted.If the Elan instance is
 192          * deleted, there is no need to explicitly delete the elan interfaces
 193          */
 194         if (elanInfo == null) {
 195             return;
 196         }
 197         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 198         if (interfaceInfo == null && elanInfo.isExternal()) {
 199             // In deleting external network, the underlying ietf Inteface might have been removed
 200             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 201             // from Operational DS instead
 202             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 203         }
 204         String elanInstanceName = elanInfo.getElanInstanceName();
 205         DataStoreJobCoordinator coordinator = DataStoreJobCoordinator.getInstance();
 206         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 207                 interfaceName, interfaceInfo, false, this);
 208         coordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 209     }
 210
 211     public void removeElanInterface(List<ListenableFuture<Void>> futures, ElanInstance elanInfo, String interfaceName,
 212             InterfaceInfo interfaceInfo, boolean isInterfaceStateRemoved) {
 213         String elanName = elanInfo.getElanInstanceName();
 214         boolean isLastElanInterface = false;
 215         boolean isLastInterfaceOnDpn = false;
 216         BigInteger dpId = null;
 217         long elanTag = elanInfo.getElanTag();
 218         WriteTransaction tx = broker.newWriteOnlyTransaction();
 219         WriteTransaction deleteFlowGroupTx = broker.newWriteOnlyTransaction();
 220         Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, tx);
 221         if (elanState == null) {
 222             return;
 223         }
 224         List<String> elanInterfaces = elanState.getElanInterfaces();
 225         if (elanInterfaces.isEmpty()) {
 226             isLastElanInterface = true;
 227         }
 228         if (interfaceInfo != null) {
 229             dpId = interfaceInfo.getDpId();
 230             DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, dpId,
 231                     interfaceName, elanTag, tx);
 232             /*
 233              * If there are not elan ports, remove the unknown dmac, terminating
 234              * service table flows, remote/local bc group
 235              */
 236             if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 237                     || dpnInterfaces.getInterfaces().isEmpty()) {
 238                 // No more Elan Interfaces in this DPN
 239                 LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(), dpId);
 240                 if (!elanUtils.isOpenStackVniSemanticsEnforced()) {
 241                     removeDefaultTermFlow(dpId, elanInfo.getElanTag());
 242                 }
 243                 removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, elanInfo.getElanTag());
 244                 removeEtreeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx);
 245                 removeElanBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 246                 removeLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 247                 removeEtreeBroadcastGrups(elanInfo, interfaceInfo, deleteFlowGroupTx);
 248                 if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 249                     if (elanUtils.isOpenStackVniSemanticsEnforced()) {
 250                         elanUtils.removeTerminatingServiceAction(dpId, elanInfo.getSegmentationId().intValue());
 251                     }
 252                     unsetExternalTunnelTable(dpId, elanInfo);
 253                 }
 254                 isLastInterfaceOnDpn = true;
 255             } else {
 256                 setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 257             }
 258         }
 259         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 260         futures.add(ElanUtils.waitForTransactionToComplete(deleteFlowGroupTx));
 261
 262         if (isLastInterfaceOnDpn && dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 263             setElanAndEtreeBCGrouponOtherDpns(elanInfo, dpId);
 264         }
 265         DataStoreJobCoordinator coordinator = DataStoreJobCoordinator.getInstance();
 266         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 267                 interfaceName, elanInfo, interfaceInfo, isInterfaceStateRemoved, this, isLastElanInterface);
 268         coordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 269                 ElanConstants.JOB_MAX_RETRIES);
 270     }
 271
 272     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 273             WriteTransaction deleteFlowGroupTx) {
 274         EtreeLeafTagName etreeLeafTag = elanUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 275         if (etreeLeafTag != null) {
 276             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 277             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 278         }
 279     }
 280
 281     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 282             WriteTransaction deleteFlowGroupTx) {
 283         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 284         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 285     }
 286
 287     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 288             WriteTransaction deleteFlowGroupTx) {
 289         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 290         if (etreeInstance != null) {
 291             BigInteger dpnId = interfaceInfo.getDpId();
 292             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 293             List<Bucket> listBuckets = new ArrayList<>();
 294             int bucketId = 0;
 295             listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
 296             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 297                     MDSALUtil.buildBucketLists(listBuckets));
 298             LOG.trace("deleted the localBroadCast Group:{}", group);
 299             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 300         }
 301     }
 302
 303     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 304             WriteTransaction deleteFlowGroupTx) {
 305         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 306         if (etreeInstance != null) {
 307             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 308             int bucketId = 0;
 309             int actionKey = 0;
 310             List<Bucket> listBuckets = new ArrayList<>();
 311             List<Action> listAction = new ArrayList<>();
 312             listAction.add(new ActionGroup(ElanUtils.getEtreeLeafLocalBCGId(etreeTag)).buildAction(++actionKey));
 313             listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
 314                     MDSALUtil.WATCH_GROUP));
 315             bucketId++;
 316             listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, etreeTag));
 317             BigInteger dpnId = interfaceInfo.getDpId();
 318             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 319             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 320                     MDSALUtil.buildBucketLists(listBuckets));
 321             LOG.trace("deleting the remoteBroadCast group:{}", group);
 322             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 323         }
 324     }
 325
 326     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName, WriteTransaction tx) {
 327         String elanName = elanInfo.getElanInstanceName();
 328         Elan elanState = ElanUtils.getElanByName(broker, elanName);
 329         if (elanState == null) {
 330             return elanState;
 331         }
 332         List<String> elanInterfaces = elanState.getElanInterfaces();
 333         boolean isRemoved = elanInterfaces.remove(interfaceName);
 334         if (!isRemoved) {
 335             return elanState;
 336         }
 337
 338         if (elanInterfaces.isEmpty()) {
 339             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName));
 340             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanMacTableOperationalDataPath(elanName));
 341             tx.delete(LogicalDatastoreType.OPERATIONAL,
 342                     ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 343         } else {
 344             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 345                     .setKey(new ElanKey(elanName)).build();
 346             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName),
 347                     updateElanState);
 348         }
 349         return elanState;
 350     }
 351
 352     private void deleteElanInterfaceFromConfigDS(String interfaceName, WriteTransaction tx) {
 353         // removing the ElanInterface from the config data_store if interface is
 354         // not present in Interface config DS
 355         if (interfaceManager.getInterfaceInfoFromConfigDataStore(interfaceName) == null
 356                 && ElanUtils.getElanInterfaceByElanInterfaceName(broker, interfaceName) != null) {
 357             tx.delete(LogicalDatastoreType.CONFIGURATION,
 358                     ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 359         }
 360     }
 361
 362     void removeEntriesForElanInterface(List<ListenableFuture<Void>> futures, ElanInstance elanInfo, InterfaceInfo
 363             interfaceInfo, String interfaceName, boolean isInterfaceStateRemoved, boolean isLastElanInterface) {
 364         String elanName = elanInfo.getElanInstanceName();
 365         WriteTransaction tx = broker.newWriteOnlyTransaction();
 366         WriteTransaction deleteFlowGroupTx = broker.newWriteOnlyTransaction();
 367         InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 368                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 369         Optional<ElanInterfaceMac> existingElanInterfaceMac = elanUtils.read(broker,
 370                 LogicalDatastoreType.OPERATIONAL, elanInterfaceId);
 371         LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 372         if (interfaceInfo != null) {
 373             if (existingElanInterfaceMac.isPresent()) {
 374                 List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 375                 List<MacEntry> macEntries = new ArrayList<>();
 376                 if (existingMacEntries != null && !existingMacEntries.isEmpty()) {
 377                     macEntries.addAll(existingMacEntries);
 378                 }
 379                 List<PhysAddress> macAddresses = macEntries.stream().map(macEntry -> {
 380                     PhysAddress macAddress = macEntry.getMacAddress();
 381                     LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 382                             macAddress.getValue(), interfaceName);
 383                     Optional<MacEntry> macEntryOptional = elanUtils.getMacEntryForElanInstance(elanName,
 384                             macAddress);
 385                     if (!isLastElanInterface && macEntryOptional.isPresent()) {
 386                         tx.delete(LogicalDatastoreType.OPERATIONAL,
 387                                 ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 388                     }
 389                     elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, deleteFlowGroupTx);
 390                     return macAddress;
 391                 }).collect(Collectors.toList());
 392
 393                 // Removing all those MACs from External Devices belonging
 394                 // to this ELAN
 395                 if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 396                     elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 397                 }
 398             }
 399             removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 400             removeFilterEqualsTable(elanInfo, interfaceInfo, deleteFlowGroupTx);
 401         } else if (existingElanInterfaceMac.isPresent()) {
 402             // Interface does not exist in ConfigDS, so lets remove everything
 403             // about that interface related to Elan
 404             List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 405             if (macEntries != null) {
 406                 for (MacEntry macEntry : macEntries) {
 407                     PhysAddress macAddress = macEntry.getMacAddress();
 408                     if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 409                         tx.delete(LogicalDatastoreType.OPERATIONAL,
 410                                 ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 411                     }
 412                 }
 413             }
 414         }
 415         if (existingElanInterfaceMac.isPresent()) {
 416             tx.delete(LogicalDatastoreType.OPERATIONAL, elanInterfaceId);
 417         }
 418         if (!isInterfaceStateRemoved) {
 419             unbindService(interfaceName, tx);
 420         }
 421         deleteElanInterfaceFromConfigDS(interfaceName, tx);
 422         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 423         futures.add(ElanUtils.waitForTransactionToComplete(deleteFlowGroupTx));
 424     }
 425
 426     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 427                                                                          String interfaceName, long elanTag,
 428                                                                          WriteTransaction tx) {
 429         DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 430         if (dpnInterfaces != null) {
 431             List<String> interfaceLists = dpnInterfaces.getInterfaces();
 432             interfaceLists.remove(interfaceName);
 433
 434             if (interfaceLists == null || interfaceLists.isEmpty()) {
 435                 deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 436                 deleteElanDpnInterface(elanName, dpId, tx);
 437             } else {
 438                 dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 439             }
 440         }
 441         return dpnInterfaces;
 442     }
 443
 444     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 445         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 446         for (DpnInterfaces dpnInterface : dpnInterfaces) {
 447             BigInteger currentDpId = dpnInterface.getDpId();
 448             if (!currentDpId.equals(dpId)) {
 449                 for (String elanInterface : dpnInterface.getInterfaces()) {
 450                     ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 451                     if (macs == null || macs.getMacEntry() == null) {
 452                         continue;
 453                     }
 454                     for (MacEntry mac : macs.getMacEntry()) {
 455                         removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 456                         removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 457                     }
 458                 }
 459             }
 460         }
 461     }
 462
 463     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 464         EtreeLeafTagName etreeLeafTag = elanUtils.getEtreeLeafTagByElanTag(elanTag);
 465         if (etreeLeafTag != null) {
 466             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac);
 467         }
 468     }
 469
 470     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 471         mdsalManager
 472                 .removeFlow(dpId,
 473                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 474                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 475                                         mac.getMacAddress().getValue(), elanTag)));
 476     }
 477
 478     /*
 479     * Possible Scenarios for update
 480     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 481         then all {1,2,3,4} should be removed
 482
 483         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 484         then all {1,2,3,4} should be added
 485
 486         c. if orig = {1,2,3,4} updated={2,3,4}
 487         then 1 should be removed
 488
 489         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 490         then we should just add 5
 491
 492         e. if orig = {1,2,3,4} updated={2,3,4,5}
 493         then 1 should be removed , 5 should be added
 494     * */
 495     @Override
 496     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 497         // updating the static-Mac Entries for the existing elanInterface
 498         String elanName = update.getElanInstanceName();
 499         String interfaceName = update.getName();
 500
 501         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 502         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 503         List<StaticMacEntries> deletedEntries = elanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 504         List<StaticMacEntries> updatedEntries = elanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 505
 506         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 507                 deletedEntry.getMacAddress()));
 508
 509         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 510         * Hence add the macentries for the same.*/
 511         for (StaticMacEntries staticMacEntry : updatedEntries) {
 512             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 513                     staticMacEntry.getMacAddress());
 514             Optional<MacEntry> existingMacEntry = elanUtils.read(broker,
 515                     LogicalDatastoreType.OPERATIONAL, macEntryIdentifier);
 516             WriteTransaction tx = broker.newWriteOnlyTransaction();
 517             if (existingMacEntry.isPresent()) {
 518                 elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 519                         elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 520                         tx);
 521             } else {
 522                 elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 523                         ElanUtils.getElanInstanceByName(broker, elanName), interfaceName, staticMacEntry, tx);
 524             }
 525             ElanUtils.waitForTransactionToComplete(tx);
 526         }
 527     }
 528
 529     @Override
 530     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 531         String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 532         String interfaceName = elanInterfaceAdded.getName();
 533         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 534         if (interfaceInfo == null) {
 535             LOG.warn("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 536             return;
 537         }
 538         ElanInstance elanInstance = ElanUtils.getElanInstanceByName(broker, elanInstanceName);
 539
 540         if (elanInstance == null) {
 541             elanInstance = new ElanInstanceBuilder().setElanInstanceName(elanInstanceName)
 542                     .setDescription(elanInterfaceAdded.getDescription()).build();
 543             // Add the ElanInstance in the Configuration data-store
 544             WriteTransaction tx = broker.newWriteOnlyTransaction();
 545             List<String> elanInterfaces = new ArrayList<>();
 546             elanInterfaces.add(interfaceName);
 547             ElanUtils.updateOperationalDataStore(broker, idManager,
 548                     elanInstance, elanInterfaces, tx);
 549             ElanUtils.waitForTransactionToComplete(tx);
 550             elanInstance = ElanUtils.getElanInstanceByName(broker, elanInstanceName);
 551         }
 552
 553         Long elanTag = elanInstance.getElanTag();
 554         // If elan tag is not updated, then put the elan interface into
 555         // unprocessed entry map and entry. Let entries
 556         // in this map get processed during ELAN update DCN.
 557         if (elanTag == null) {
 558             ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 559             if (elanInterfaces == null) {
 560                 elanInterfaces = new ConcurrentLinkedQueue<>();
 561             }
 562             elanInterfaces.add(elanInterfaceAdded);
 563             unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 564             return;
 565         }
 566         DataStoreJobCoordinator coordinator = DataStoreJobCoordinator.getInstance();
 567         InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 568                 interfaceInfo, elanInstance, this);
 569         coordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 570     }
 571
 572     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) throws ElanException {
 573         List<ListenableFuture<Void>> futures = new ArrayList<>();
 574         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 575         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 576             return futures;
 577         }
 578         for (ElanInterface elanInterface : elanInterfaces) {
 579             String interfaceName = elanInterface.getName();
 580             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 581             addElanInterface(futures, elanInterface, interfaceInfo, elanInstance);
 582         }
 583         return futures;
 584     }
 585
 586     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 587             WriteTransaction writeFlowGroupTx) throws ElanException {
 588         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 589                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 590         List<DpnInterfaces> dpnInterfaceLists = null;
 591         if (elanDpnInterfacesList != null) {
 592             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 593         }
 594         if (dpnInterfaceLists == null) {
 595             dpnInterfaceLists = new ArrayList<>();
 596         }
 597         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 598             BigInteger dstDpId = interfaceInfo.getDpId();
 599             if (dpnInterfaces.getDpId().equals(dstDpId)) {
 600                 continue;
 601             }
 602             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 603             for (String remoteIf : remoteElanInterfaces) {
 604                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 605                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 606                 if (elanIfMac == null) {
 607                     continue;
 608                 }
 609                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 610                 if (remoteMacEntries != null) {
 611                     for (MacEntry macEntry : remoteMacEntries) {
 612                         String macAddress = macEntry.getMacAddress().getValue();
 613                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 614                                 dstDpId, elanInstance.getElanInstanceName());
 615                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 616                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 617                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 618                     }
 619                 }
 620             }
 621         }
 622     }
 623
 624     void addElanInterface(List<ListenableFuture<Void>> futures, ElanInterface elanInterface,
 625             InterfaceInfo interfaceInfo, ElanInstance elanInstance) throws ElanException {
 626         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 627         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 628         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 629
 630         String interfaceName = elanInterface.getName();
 631         String elanInstanceName = elanInterface.getElanInstanceName();
 632
 633         Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 634         WriteTransaction tx = broker.newWriteOnlyTransaction();
 635         if (elanInfo == null) {
 636             List<String> elanInterfaces = new ArrayList<>();
 637             elanInterfaces.add(interfaceName);
 638             ElanUtils.updateOperationalDataStore(broker, idManager,
 639                     elanInstance, elanInterfaces, tx);
 640         } else {
 641             createElanStateList(elanInstanceName, interfaceName, tx);
 642         }
 643         boolean isFirstInterfaceInDpn = false;
 644         // Specific actions to the DPN where the ElanInterface has been added,
 645         // for example, programming the
 646         // External tunnel table if needed or adding the ElanInterface to the
 647         // DpnInterfaces in the operational DS.
 648         BigInteger dpId = interfaceInfo != null ? dpId = interfaceInfo.getDpId() : null;
 649         DpnInterfaces dpnInterfaces = null;
 650         if (dpId != null && !dpId.equals(ElanConstants.INVALID_DPN)) {
 651             InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 652                     .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
 653             Optional<DpnInterfaces> existingElanDpnInterfaces = elanUtils.read(broker,
 654                     LogicalDatastoreType.OPERATIONAL, elanDpnInterfaces);
 655             if (!existingElanDpnInterfaces.isPresent()) {
 656                 isFirstInterfaceInDpn = true;
 657                 // ELAN's 1st ElanInterface added to this DPN
 658                 dpnInterfaces = createElanInterfacesList(elanInstanceName, interfaceName, dpId, tx);
 659                 // The 1st ElanInterface in a DPN must program the Ext Tunnel
 660                 // table, but only if Elan has VNI
 661                 if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 662                     setExternalTunnelTable(dpId, elanInstance);
 663                 }
 664                 elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 665             } else {
 666                 List<String> elanInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 667                 elanInterfaces.add(interfaceName);
 668                 if (elanInterfaces.size() == 1) { // 1st dpn interface
 669                     elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 670                 }
 671                 dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, dpId, elanInterfaces, tx);
 672             }
 673         }
 674
 675         // add code to install Local/Remote BC group, unknow DMAC entry,
 676         // terminating service table flow entry
 677         // call bindservice of interfacemanager to create ingress table flow
 678         // enty.
 679         // Add interface to the ElanInterfaceForwardingEntires Container
 680         createElanInterfaceTablesList(interfaceName, tx);
 681         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 682         if (interfaceInfo != null) {
 683             installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, dpnInterfaces,
 684                     isFirstInterfaceInDpn, null);
 685
 686             // add the vlan provider interface to remote BC group for the elan
 687             // for internal vlan networks
 688             if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 689                 if (interfaceManager.isExternalInterface(interfaceName)) {
 690                     LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 691                     handleExternalInterfaceEvent(elanInstance, dpnInterfaces, dpId);
 692                 }
 693             }
 694         }
 695
 696         if (isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 697             //update the remote-DPNs remoteBC group entry with Tunnels
 698             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance, dpId);
 699             setElanAndEtreeBCGrouponOtherDpns(elanInstance, dpId);
 700         }
 701
 702         DataStoreJobCoordinator coordinator = DataStoreJobCoordinator.getInstance();
 703         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 704         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 705                 elanInterface, interfaceInfo, elanInstance, isFirstInterfaceInDpn, this);
 706         coordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 707     }
 708
 709     void setupEntriesForElanInterface(List<ListenableFuture<Void>> futures, ElanInstance elanInstance,
 710             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn)
 711             throws ElanException {
 712         String elanInstanceName = elanInstance.getElanInstanceName();
 713         String interfaceName = elanInterface.getName();
 714         WriteTransaction tx = broker.newWriteOnlyTransaction();
 715         BigInteger dpId = interfaceInfo.getDpId();
 716         WriteTransaction writeFlowGroupTx = broker.newWriteOnlyTransaction();
 717         installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 718                 isFirstInterfaceInDpn, tx, writeFlowGroupTx);
 719
 720         List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 721         List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 722
 723         boolean isInterfaceOperational = isOperational(interfaceInfo);
 724         if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 725             for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 726                 InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 727                         staticMacEntry.getMacAddress());
 728                 Optional<MacEntry> existingMacEntry = elanUtils.read(broker,
 729                         LogicalDatastoreType.OPERATIONAL, macId);
 730                 if (existingMacEntry.isPresent()) {
 731                     elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 732                             elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 733                             existingMacEntry.get(), tx);
 734                 } else {
 735                     elanForwardingEntriesHandler
 736                             .addElanInterfaceForwardingTableList(elanInstance, interfaceName, staticMacEntry, tx);
 737                 }
 738
 739                 if (isInterfaceOperational) {
 740                     // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 741                     // DPNs
 742                     String macAddress = staticMacEntry.getMacAddress().getValue();
 743                     LOG.info("programming smac and dmacs for {} on source and other DPNs for elan {} and interface {}",
 744                             macAddress, elanInstanceName, interfaceName);
 745                     elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 746                             staticMacEntry.getMacAddress().getValue(), true, writeFlowGroupTx);
 747                 }
 748             }
 749
 750             if (isInterfaceOperational) {
 751                 // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 752                 // on purpose.
 753                 for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 754                     staticMacAddresses.add(staticMacEntry.getMacAddress());
 755                 }
 756                 elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 757                         staticMacAddresses);
 758             }
 759         }
 760         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 761         futures.add(ElanUtils.waitForTransactionToComplete(writeFlowGroupTx));
 762         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 763             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 764             try {
 765                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 766                 if (neutronPort != null) {
 767                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 768                 }
 769             } catch (IllegalArgumentException ex) {
 770                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 771             }
 772         }
 773     }
 774
 775     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 776             PhysAddress physAddress) {
 777         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 778         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 779         Optional<MacEntry> existingMacEntry = elanUtils.read(broker,
 780                 LogicalDatastoreType.OPERATIONAL, macId);
 781
 782         if (!existingMacEntry.isPresent()) {
 783             return;
 784         }
 785
 786         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 787                 .setKey(new MacEntryKey(physAddress)).build();
 788         WriteTransaction tx = broker.newWriteOnlyTransaction();
 789         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 790                 ElanUtils.getElanInstanceByName(broker, elanInstanceName), interfaceInfo, macEntry, tx);
 791         ElanUtils.waitForTransactionToComplete(tx);
 792     }
 793
 794     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 795         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 796                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 797     }
 798
 799     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 800             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, WriteTransaction tx,
 801             WriteTransaction writeFlowGroupTx) throws ElanException {
 802         if (!isOperational(interfaceInfo)) {
 803             return;
 804         }
 805         BigInteger dpId = interfaceInfo.getDpId();
 806         if (!elanUtils.isOpenStackVniSemanticsEnforced()) {
 807             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, writeFlowGroupTx);
 808         }
 809         setupFilterEqualsTable(elanInstance, interfaceInfo, writeFlowGroupTx);
 810         if (isFirstInterfaceInDpn) {
 811             // Terminating Service , UnknownDMAC Table.
 812             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 813             // for ELAN Instance is VxLAN
 814             if (isVxlan(elanInstance)) {
 815                 setupTerminateServiceTable(elanInstance, dpId, writeFlowGroupTx);
 816             }
 817             setupUnknownDMacTable(elanInstance, dpId, writeFlowGroupTx);
 818             /*
 819              * Install remote DMAC flow. This is required since this DPN is
 820              * added later to the elan instance and remote DMACs of other
 821              * interfaces in this elan instance are not present in the current
 822              * dpn.
 823              */
 824             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 825                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 826                         elanInstance.getElanInstanceName());
 827                 programRemoteDmacFlow(elanInstance, interfaceInfo, writeFlowGroupTx);
 828             }
 829         }
 830         // bind the Elan service to the Interface
 831         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), tx);
 832     }
 833
 834     public void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 835             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, WriteTransaction tx) {
 836         if (!isOperational(interfaceInfo)) {
 837             return;
 838         }
 839         // LocalBroadcast Group creation with elan-Interfaces
 840         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 841         if (isFirstInterfaceInDpn) {
 842             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 843             BigInteger dpId = interfaceInfo.getDpId();
 844             // RemoteBroadcast Group creation
 845             try {
 846                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 847             } catch (InterruptedException e1) {
 848                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 849             }
 850             setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId);
 851             try {
 852                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 853             } catch (InterruptedException e1) {
 854                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 855             }
 856         }
 857     }
 858
 859     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 860             WriteTransaction writeFlowGroupTx) {
 861         int ifTag = interfaceInfo.getInterfaceTag();
 862         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 863                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag), 9, elanInfo.getElanInstanceName(), 0, 0,
 864                 ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 865                 elanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 866                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 867
 868         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flow, writeFlowGroupTx);
 869
 870         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 871                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, 1000 + ifTag), 10, elanInfo.getElanInstanceName(), 0,
 872                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 873                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 874
 875         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flowEntry, writeFlowGroupTx);
 876     }
 877
 878     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 879             WriteTransaction deleteFlowGroupTx) {
 880         int ifTag = interfaceInfo.getInterfaceTag();
 881         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 882                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag), 9, elanInfo.getElanInstanceName(), 0, 0,
 883                 ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 884                 elanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 885                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 886
 887         mdsalManager.removeFlowToTx(interfaceInfo.getDpId(), flow, deleteFlowGroupTx);
 888
 889         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 890                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, 1000 + ifTag), 10, elanInfo.getElanInstanceName(), 0,
 891                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 892                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 893
 894         mdsalManager.removeFlowToTx(interfaceInfo.getDpId(), flowEntity, deleteFlowGroupTx);
 895     }
 896
 897     private List<Bucket> getRemoteBCGroupBucketInfos(ElanInstance elanInfo, int bucketKeyStart,
 898                                                      InterfaceInfo interfaceInfo, long elanTag) {
 899         return getRemoteBCGroupBuckets(elanInfo, null, interfaceInfo.getDpId(), bucketKeyStart, elanTag);
 900     }
 901
 902     public List<Bucket> getRemoteBCGroupBuckets(ElanInstance elanInfo, DpnInterfaces dpnInterfaces, BigInteger dpnId,
 903                                                 int bucketId, long elanTag) {
 904         List<Bucket> listBucketInfo = new ArrayList<>();
 905         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
 906
 907         if (isVxlan(elanInfo)) {
 908             listBucketInfo.addAll(getRemoteBCGroupTunnelBuckets(elanDpns, dpnId, bucketId,
 909                     elanUtils.isOpenStackVniSemanticsEnforced() ? elanInfo.getSegmentationId() : elanTag));
 910         }
 911         listBucketInfo.addAll(getRemoteBCGroupExternalPortBuckets(elanDpns, dpnInterfaces, dpnId,
 912             getNextAvailableBucketId(listBucketInfo.size())));
 913         listBucketInfo.addAll(getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnId,
 914             getNextAvailableBucketId(listBucketInfo.size())));
 915         listBucketInfo.addAll(getRemoteBCGroupBucketsOfElanExternalTeps(elanInfo, dpnId,
 916                 getNextAvailableBucketId(listBucketInfo.size())));
 917         return listBucketInfo;
 918     }
 919
 920     private int getNextAvailableBucketId(int bucketSize) {
 921         return bucketSize + 1;
 922     }
 923
 924     @SuppressWarnings("checkstyle:IllegalCatch")
 925     private List<Bucket> getRemoteBCGroupTunnelBuckets(ElanDpnInterfacesList elanDpns, BigInteger dpnId, int bucketId,
 926             long elanTagOrVni) {
 927         List<Bucket> listBucketInfo = new ArrayList<>();
 928         if (elanDpns != null) {
 929             for (DpnInterfaces dpnInterface : elanDpns.getDpnInterfaces()) {
 930                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpnId)
 931                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
 932                     try {
 933                         List<Action> listActionInfo = elanUtils.getInternalTunnelItmEgressAction(dpnId,
 934                                 dpnInterface.getDpId(), elanTagOrVni);
 935                         if (listActionInfo.isEmpty()) {
 936                             continue;
 937                         }
 938                         listBucketInfo.add(MDSALUtil.buildBucket(listActionInfo, MDSALUtil.GROUP_WEIGHT, bucketId,
 939                                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
 940                         bucketId++;
 941                     } catch (Exception ex) {
 942                         LOG.error("Logical Group Interface not found between source Dpn - {}, destination Dpn - {} ",
 943                                 dpnId, dpnInterface.getDpId(), ex);
 944                     }
 945                 }
 946             }
 947         }
 948         return listBucketInfo;
 949     }
 950
 951     private List<Bucket> getRemoteBCGroupExternalPortBuckets(ElanDpnInterfacesList elanDpns,
 952             DpnInterfaces dpnInterfaces, BigInteger dpnId, int bucketId) {
 953         DpnInterfaces currDpnInterfaces = dpnInterfaces != null ? dpnInterfaces : getDpnInterfaces(elanDpns, dpnId);
 954         if (currDpnInterfaces == null || !elanUtils.isDpnPresent(currDpnInterfaces.getDpId())
 955                 || currDpnInterfaces.getInterfaces() == null || currDpnInterfaces.getInterfaces().isEmpty()) {
 956             return Collections.emptyList();
 957         }
 958         List<Bucket> listBucketInfo = new ArrayList<>();
 959         for (String interfaceName : currDpnInterfaces.getInterfaces()) {
 960             if (interfaceManager.isExternalInterface(interfaceName)) {
 961                 List<Action> listActionInfo = elanUtils.getExternalPortItmEgressAction(interfaceName);
 962                 if (!listActionInfo.isEmpty()) {
 963                     listBucketInfo.add(MDSALUtil.buildBucket(listActionInfo, MDSALUtil.GROUP_WEIGHT, bucketId,
 964                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
 965                     bucketId++;
 966                 }
 967             }
 968         }
 969         return listBucketInfo;
 970     }
 971
 972     private DpnInterfaces getDpnInterfaces(ElanDpnInterfacesList elanDpns, BigInteger dpnId) {
 973         if (elanDpns != null) {
 974             for (DpnInterfaces dpnInterface : elanDpns.getDpnInterfaces()) {
 975                 if (dpnInterface.getDpId().equals(dpnId)) {
 976                     return dpnInterface;
 977                 }
 978             }
 979         }
 980         return null;
 981     }
 982
 983     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId) {
 984         int elanTag = elanInfo.getElanTag().intValue();
 985         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
 986         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId);
 987         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 988         if (etreeInstance != null) {
 989             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
 990             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
 991             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId);
 992         }
 993     }
 994
 995     @SuppressWarnings("checkstyle:IllegalCatch")
 996     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId) {
 997         int bucketId = 0;
 998         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
 999         if (elanDpns != null) {
1000             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1001             for (DpnInterfaces dpnInterface : dpnInterfaces) {
1002                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
1003                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
1004                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
1005                     List<Action> listAction = new ArrayList<>();
1006                     int actionKey = 0;
1007                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1008                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
1009                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1010                     bucketId++;
1011                     for (DpnInterfaces otherFes : dpnInterfaces) {
1012                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
1013                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
1014                             && !otherFes.getInterfaces().isEmpty()) {
1015                             try {
1016                                 List<Action> remoteListActionInfo = elanUtils.getInternalTunnelItmEgressAction(
1017                                         dpnInterface.getDpId(), otherFes.getDpId(),
1018                                         elanUtils.isOpenStackVniSemanticsEnforced() ? elanInfo.getSegmentationId()
1019                                                 : elanTag);
1020                                 if (!remoteListActionInfo.isEmpty()) {
1021                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
1022                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1023                                     bucketId++;
1024                                 }
1025                             } catch (Exception ex) {
1026                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1027                                         + "Logical Group Interface not found between source Dpn - {}, "
1028                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1029                                 return;
1030                             }
1031                         }
1032                     }
1033                     List<Bucket> elanL2GwDevicesBuckets = getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpId,
1034                             bucketId);
1035                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1036
1037                     if (remoteListBucketInfo.isEmpty()) {
1038                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1039                         continue;
1040                     }
1041                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1042                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1043                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1044                     mdsalManager.syncInstallGroup(dpnInterface.getDpId(), group,
1045                             ElanConstants.DELAY_TIME_IN_MILLISECOND);
1046                 }
1047             }
1048             try {
1049                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1050             } catch (InterruptedException e1) {
1051                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1052             }
1053         }
1054     }
1055
1056     /**
1057      * Returns the bucket info with the given interface as the only bucket.
1058      */
1059     private Bucket getLocalBCGroupBucketInfo(InterfaceInfo interfaceInfo, int bucketIdStart) {
1060         return MDSALUtil.buildBucket(getInterfacePortActions(interfaceInfo), MDSALUtil.GROUP_WEIGHT, bucketIdStart,
1061                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
1062     }
1063
1064     private List<MatchInfo> buildMatchesForVni(Long vni) {
1065         List<MatchInfo> mkMatches = new ArrayList<>();
1066         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1067         mkMatches.add(match);
1068         return mkMatches;
1069     }
1070
1071     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1072         List<InstructionInfo> mkInstructions = new ArrayList<>();
1073         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1074         return mkInstructions;
1075     }
1076
1077     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1078         List<MatchInfo> mkMatches = new ArrayList<>();
1079         // Matching metadata
1080         mkMatches.add(new MatchMetadata(
1081                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1082         return mkMatches;
1083     }
1084
1085     /**
1086      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1087      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1088      *
1089      * @param elanTag
1090      *            elanTag to be written in metadata when flow is selected
1091      * @return the instructions ready to be installed in a flow
1092      */
1093     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1094         List<InstructionInfo> mkInstructions = new ArrayList<>();
1095         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1096                 .getElanMetadataMask()));
1097         /* applicable for EXTERNAL_TUNNEL_TABLE only
1098         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1099         */
1100         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1101         return mkInstructions;
1102     }
1103
1104     // Install DMAC entry on dst DPN
1105     public void installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo, BigInteger dstDpId,
1106             List<ListenableFuture<Void>> futures) throws ElanException {
1107         String interfaceName = interfaceInfo.getInterfaceName();
1108         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1109         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1110             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1111             WriteTransaction writeFlowTx = broker.newWriteOnlyTransaction();
1112             for (MacEntry macEntry : macEntries) {
1113                 String macAddress = macEntry.getMacAddress().getValue();
1114                 LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress, interfaceName);
1115                 synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1116                         interfaceInfo.getDpId())) {
1117                     LOG.info("Acquired lock for mac : " + macAddress + ". Proceeding with remote dmac"
1118                             + " install operation.");
1119                     elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress,
1120                             writeFlowTx);
1121                 }
1122             }
1123             futures.add(ElanUtils.waitForTransactionToComplete(writeFlowTx));
1124         }
1125     }
1126
1127     public void setupElanBroadcastGroups(ElanInstance elanInfo, BigInteger dpnId) {
1128         setupElanBroadcastGroups(elanInfo, null, dpnId);
1129     }
1130
1131     public void setupElanBroadcastGroups(ElanInstance elanInfo, DpnInterfaces dpnInterfaces, BigInteger dpnId) {
1132         setupStandardElanBroadcastGroups(elanInfo, dpnInterfaces, dpnId);
1133         setupLeavesEtreeBroadcastGroups(elanInfo, dpnInterfaces, dpnId);
1134     }
1135
1136     public void setupStandardElanBroadcastGroups(ElanInstance elanInfo, DpnInterfaces dpnInterfaces, BigInteger dpnId) {
1137         List<Bucket> listBucket = new ArrayList<>();
1138         int bucketId = 0;
1139         int actionKey = 0;
1140         Long elanTag = elanInfo.getElanTag();
1141         List<Action> listAction = new ArrayList<>();
1142         listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1143         listBucket.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
1144                 MDSALUtil.WATCH_GROUP));
1145         bucketId++;
1146         List<Bucket> listBucketInfoRemote = getRemoteBCGroupBuckets(elanInfo, dpnInterfaces, dpnId, bucketId, elanTag);
1147         listBucket.addAll(listBucketInfoRemote);
1148         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
1149         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1150                 MDSALUtil.buildBucketLists(listBucket));
1151         LOG.trace("Installing the remote BroadCast Group:{}", group);
1152         mdsalManager.syncInstallGroup(dpnId, group, ElanConstants.DELAY_TIME_IN_MILLISECOND);
1153     }
1154
1155     public void setupLeavesEtreeBroadcastGroups(ElanInstance elanInfo, DpnInterfaces dpnInterfaces, BigInteger dpnId) {
1156         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1157         if (etreeInstance != null) {
1158             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1159             List<Bucket> listBucket = new ArrayList<>();
1160             int bucketId = 0;
1161             int actionKey = 0;
1162             List<Action> listAction = new ArrayList<>();
1163             listAction.add(new ActionGroup(ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag)).buildAction(++actionKey));
1164             listBucket.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
1165                     MDSALUtil.WATCH_GROUP));
1166             bucketId++;
1167             List<Bucket> listBucketInfoRemote = getRemoteBCGroupBuckets(elanInfo, dpnInterfaces, dpnId, bucketId,
1168                     etreeLeafTag);
1169             listBucket.addAll(listBucketInfoRemote);
1170             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
1171             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1172                     MDSALUtil.buildBucketLists(listBucket));
1173             LOG.trace("Installing the remote BroadCast Group:{}", group);
1174             mdsalManager.syncInstallGroup(dpnId, group,
1175                     ElanConstants.DELAY_TIME_IN_MILLISECOND);
1176         }
1177     }
1178
1179     private void createDropBucket(List<Bucket> listBucket) {
1180         List<Action> actionsInfos = new ArrayList<>();
1181         actionsInfos.add(new ActionDrop().buildAction());
1182         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1183                 MDSALUtil.WATCH_GROUP);
1184         listBucket.add(dropBucket);
1185     }
1186
1187     public void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1188             InterfaceInfo interfaceInfo) {
1189         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1190         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1191     }
1192
1193     public void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1194             InterfaceInfo interfaceInfo) {
1195         List<Bucket> listBucket = new ArrayList<>();
1196         int bucketId = 0;
1197         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1198
1199         List<String> interfaces = new ArrayList<>();
1200         if (newDpnInterface != null) {
1201             interfaces = newDpnInterface.getInterfaces();
1202         }
1203         for (String ifName : interfaces) {
1204             // In case if there is a InterfacePort in the cache which is not in
1205             // operational state, skip processing it
1206             InterfaceInfo ifInfo = interfaceManager
1207                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1208             if (!isOperational(ifInfo)) {
1209                 continue;
1210             }
1211
1212             if (!interfaceManager.isExternalInterface(ifName)) {
1213                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1214                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1215                 bucketId++;
1216             }
1217         }
1218
1219         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1220                 MDSALUtil.buildBucketLists(listBucket));
1221         LOG.trace("installing the localBroadCast Group:{}", group);
1222         mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group,
1223                 ElanConstants.DELAY_TIME_IN_MILLISECOND);
1224     }
1225
1226     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1227             InterfaceInfo interfaceInfo) {
1228         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1229         if (etreeInstance != null) {
1230             List<Bucket> listBucket = new ArrayList<>();
1231             int bucketId = 0;
1232
1233             List<String> interfaces = new ArrayList<>();
1234             if (newDpnInterface != null) {
1235                 interfaces = newDpnInterface.getInterfaces();
1236             }
1237             for (String ifName : interfaces) {
1238                 // In case if there is a InterfacePort in the cache which is not
1239                 // in
1240                 // operational state, skip processing it
1241                 InterfaceInfo ifInfo = interfaceManager
1242                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1243                 if (!isOperational(ifInfo)) {
1244                     continue;
1245                 }
1246
1247                 if (!interfaceManager.isExternalInterface(ifName)) {
1248                     // only add root interfaces
1249                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1250                 }
1251             }
1252
1253             if (listBucket.isEmpty()) { // No Buckets
1254                 createDropBucket(listBucket);
1255             }
1256
1257             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1258             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1259             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1260                     MDSALUtil.buildBucketLists(listBucket));
1261             LOG.trace("installing the localBroadCast Group:{}", group);
1262             mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group,
1263                     ElanConstants.DELAY_TIME_IN_MILLISECOND);
1264         }
1265     }
1266
1267     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1268             InterfaceInfo ifInfo) {
1269         EtreeInterface etreeInterface = ElanUtils.getEtreeInterfaceByElanInterfaceName(broker, ifName);
1270         if (etreeInterface != null && etreeInterface.getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1271             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1272                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1273             bucketId++;
1274         }
1275         return bucketId;
1276     }
1277
1278     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1279             WriteTransaction deleteFlowGroupTx) {
1280         BigInteger dpnId = interfaceInfo.getDpId();
1281         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1282         List<Bucket> listBuckets = new ArrayList<>();
1283         int bucketId = 0;
1284         listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
1285         // listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, 1,
1286         // interfaceInfo));
1287         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1288                 MDSALUtil.buildBucketLists(listBuckets));
1289         LOG.trace("deleted the localBroadCast Group:{}", group);
1290         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1291     }
1292
1293     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1294             WriteTransaction deleteFlowGroupTx) {
1295         int bucketId = 0;
1296         int actionKey = 0;
1297         Long elanTag = elanInfo.getElanTag();
1298         List<Bucket> listBuckets = new ArrayList<>();
1299         List<Action> listAction = new ArrayList<>();
1300         listAction.add(new ActionGroup(++actionKey, ElanUtils.getElanLocalBCGId(elanTag)).buildAction());
1301         listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
1302                 MDSALUtil.WATCH_GROUP));
1303         bucketId++;
1304         listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, elanTag));
1305         BigInteger dpnId = interfaceInfo.getDpId();
1306         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1307         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1308                 MDSALUtil.buildBucketLists(listBuckets));
1309         LOG.trace("deleting the remoteBroadCast group:{}", group);
1310         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1311     }
1312
1313     /**
1314      * Installs a flow in the External Tunnel table consisting in translating
1315      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1316      * elanTag that will be used later in the ELANs pipeline.
1317      *
1318      * @param dpnId
1319      *            the dpn id
1320      * @param elanInfo
1321      *            the elan info
1322      */
1323     public void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1324         long elanTag = elanInfo.getElanTag();
1325         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1326                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1327                 elanInfo.getElanInstanceName(), // flowName
1328                 0, // idleTimeout
1329                 0, // hardTimeout
1330                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1331                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1332                 getInstructionsIntOrExtTunnelTable(elanTag));
1333
1334         mdsalManager.installFlow(flowEntity);
1335     }
1336
1337     /**
1338      * Removes, from External Tunnel table, the flow that translates from VNI to
1339      * elanTag. Important: ensure this method is only called whenever there is
1340      * no other ElanInterface in the specified DPN
1341      *
1342      * @param dpnId
1343      *            DPN whose Ext Tunnel table is going to be modified
1344      * @param elanInfo
1345      *            holds the elanTag needed for selecting the flow to be removed
1346      */
1347     public void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1348         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1349         // last ElanInstance plus
1350         // adding a new one does (almost at the same time) are executed in that
1351         // exact order
1352
1353         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1354         FlowEntity flowEntity = new FlowEntityBuilder()
1355             .setDpnId(dpnId)
1356             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1357             .setFlowId(flowId)
1358             .build();
1359         mdsalManager.removeFlow(flowEntity);
1360     }
1361
1362     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1363         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1364         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1365     }
1366
1367     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1368             WriteTransaction writeFlowGroupTx) {
1369         List<? extends MatchInfoBase> listMatchInfoBase;
1370         List<InstructionInfo> instructionInfos;
1371         long serviceId;
1372         if (!elanUtils.isOpenStackVniSemanticsEnforced()) {
1373             serviceId = elanTag;
1374             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1375             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1376         } else {
1377             serviceId = elanInfo.getSegmentationId();
1378             listMatchInfoBase = buildMatchesForVni(serviceId);
1379             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1380         }
1381         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1382                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1383                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1384                 instructionInfos);
1385         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1386     }
1387
1388     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1389             WriteTransaction writeFlowGroupTx) {
1390         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1391         if (etreeInstance != null) {
1392             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1393         }
1394     }
1395
1396     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1397         long elanTag = elanInfo.getElanTag();
1398         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1399         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1400         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1401     }
1402
1403     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1404             WriteTransaction writeFlowGroupTx) {
1405         EtreeLeafTagName etreeLeafTag = elanUtils.getEtreeLeafTagByElanTag(elanTag);
1406         if (etreeLeafTag != null) {
1407             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1408             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1409             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1410         }
1411     }
1412
1413     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1414             WriteTransaction writeFlowGroupTx) {
1415         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1416                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1417                 5, elanInfo.getElanInstanceName(), 0, 0,
1418                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1419                 getMatchesForElanTag(elanTag, /* SH flag */false),
1420                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1421
1422         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1423     }
1424
1425     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1426             WriteTransaction writeFlowGroupTx) {
1427         // only if ELAN can connect to external network, perform the following
1428
1429         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1430             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1431                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1432                     5, elanInfo.getElanInstanceName(), 0, 0,
1433                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1434                     getMatchesForElanTag(elanTag, /* SH flag */true),
1435                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1436             mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1437         }
1438     }
1439
1440
1441     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo, WriteTransaction deleteFlowGroupTx,
1442             long elanTag) {
1443         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1444                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1445         mdsalManager.removeFlowToTx(dpId, flow, deleteFlowGroupTx);
1446
1447         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1448             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1449                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1450                     .build();
1451             mdsalManager.removeFlowToTx(dpId, flow2, deleteFlowGroupTx);
1452         }
1453     }
1454
1455     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1456         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1457     }
1458
1459     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag, WriteTransaction tx) {
1460         if (isStandardElanService(elanInterface)) {
1461             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1462                     elanInterface.getName(), lportTag, tx);
1463         } else { // Etree service
1464             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1465         }
1466     }
1467
1468     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1469             WriteTransaction tx) {
1470         int instructionKey = 0;
1471         List<Instruction> instructions = new ArrayList<>();
1472         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1473                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1474
1475         List<Action> actions = new ArrayList<>();
1476         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1477                 lportTag).buildAction());
1478         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1479                 elanTag).buildAction());
1480         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1481
1482         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1483                 ++instructionKey));
1484
1485         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1486         BoundServices serviceInfo = ElanUtils.getBoundServices(
1487                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1488                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1489         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1490         Optional<BoundServices> existingElanService = elanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1491                 bindServiceId);
1492         if (!existingElanService.isPresent()) {
1493             tx.put(LogicalDatastoreType.CONFIGURATION, bindServiceId, serviceInfo,
1494                     WriteTransaction.CREATE_MISSING_PARENTS);
1495         }
1496     }
1497
1498     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1499             WriteTransaction tx) {
1500         if (elanInterface.getAugmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1501             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1502                     lportTag, tx);
1503         } else {
1504             EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1505             if (etreeInstance == null) {
1506                 LOG.error("EtreeInterface " + elanInterface.getName() + " is associated with a non EtreeInstance: "
1507                         + elanInfo.getElanInstanceName());
1508             } else {
1509                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1510                         elanInterface.getName(), lportTag, tx);
1511             }
1512         }
1513     }
1514
1515     private boolean isStandardElanService(ElanInterface elanInterface) {
1516         return elanInterface.getAugmentation(EtreeInterface.class) == null;
1517     }
1518
1519     private boolean isStandardElanService(ElanInstance elanInstance) {
1520         return elanInstance.getAugmentation(EtreeInstance.class) == null;
1521     }
1522
1523     protected void unbindService(String interfaceName, WriteTransaction tx) {
1524         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1525         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1526         Optional<BoundServices> existingElanService = elanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1527                 bindServiceId);
1528         if (existingElanService.isPresent()) {
1529             tx.delete(LogicalDatastoreType.CONFIGURATION, bindServiceId);
1530         }
1531     }
1532
1533     private String getFlowRef(long tableId, long elanTag) {
1534         return String.valueOf(tableId) + elanTag;
1535     }
1536
1537     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1538         return String.valueOf(tableId) + elanTag + shFlag;
1539     }
1540
1541     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1542         List<Action> listAction = new ArrayList<>();
1543         int actionKey = 0;
1544         listAction.add(
1545             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1546         actionKey++;
1547         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1548         return listAction;
1549     }
1550
1551     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1552             List<String> interfaceNames, WriteTransaction tx) {
1553         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1554                 .setKey(new DpnInterfacesKey(dpId)).build();
1555         tx.put(LogicalDatastoreType.OPERATIONAL,
1556                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1557                 WriteTransaction.CREATE_MISSING_PARENTS);
1558         return dpnInterface;
1559     }
1560
1561     /**
1562      * Delete elan dpn interface from operational DS.
1563      *
1564      * @param elanInstanceName
1565      *            the elan instance name
1566      * @param dpId
1567      *            the dp id
1568      */
1569     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId, WriteTransaction tx) {
1570         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1571                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1572         Optional<DpnInterfaces> dpnInterfaces = elanUtils.read(broker,
1573                 LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1574         if (dpnInterfaces.isPresent()) {
1575             tx.delete(LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1576         }
1577     }
1578
1579     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1580             WriteTransaction tx) {
1581         List<String> interfaceNames = new ArrayList<>();
1582         interfaceNames.add(interfaceName);
1583         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1584                 .setKey(new DpnInterfacesKey(dpId)).build();
1585         tx.put(LogicalDatastoreType.OPERATIONAL,
1586                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1587                 WriteTransaction.CREATE_MISSING_PARENTS);
1588         return dpnInterface;
1589     }
1590
1591     private void createElanInterfaceTablesList(String interfaceName, WriteTransaction tx) {
1592         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1593                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1594         Optional<ElanInterfaceMac> interfaceMacTables = elanUtils.read(broker,
1595                 LogicalDatastoreType.OPERATIONAL, elanInterfaceMacTables);
1596         // Adding new Elan Interface Port to the operational DataStore without
1597         // Static-Mac Entries..
1598         if (!interfaceMacTables.isPresent()) {
1599             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1600                     .setKey(new ElanInterfaceMacKey(interfaceName)).build();
1601             tx.put(LogicalDatastoreType.OPERATIONAL,
1602                     ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1603                     WriteTransaction.CREATE_MISSING_PARENTS);
1604         }
1605     }
1606
1607     private void createElanStateList(String elanInstanceName, String interfaceName, WriteTransaction tx) {
1608         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1609         Optional<Elan> elanInterfaceLists = elanUtils.read(broker,
1610                 LogicalDatastoreType.OPERATIONAL, elanInstance);
1611         // Adding new Elan Interface Port to the operational DataStore without
1612         // Static-Mac Entries..
1613         if (elanInterfaceLists.isPresent()) {
1614             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1615             if (interfaceLists == null) {
1616                 interfaceLists = new ArrayList<>();
1617             }
1618             interfaceLists.add(interfaceName);
1619             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1620                     .setKey(new ElanKey(elanInstanceName)).build();
1621             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName),
1622                     elanState, WriteTransaction.CREATE_MISSING_PARENTS);
1623         }
1624     }
1625
1626     private boolean isOperational(InterfaceInfo interfaceInfo) {
1627         if (interfaceInfo == null) {
1628             return false;
1629         }
1630         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1631     }
1632
1633     @SuppressWarnings("checkstyle:IllegalCatch")
1634     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) throws ElanException {
1635         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1636         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1637                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1638         if (dpnInterfaceLists == null) {
1639             return;
1640         }
1641         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1642         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1643             int cnt = 0;
1644             String elanName = elanDpns.getElanInstanceName();
1645             ElanInstance elanInfo = ElanUtils.getElanInstanceByName(broker, elanName);
1646             if (elanInfo == null) {
1647                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1648                         + "skipping this ELAN for tunnel handling", elanName);
1649                 continue;
1650             }
1651             if (ElanUtils.isFlat(elanInfo) || ElanUtils.isVlan(elanInfo)) {
1652                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1653                 continue;
1654             }
1655             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1656             if (dpnInterfaces == null) {
1657                 continue;
1658             }
1659             DpnInterfaces dstDpnIf = null;
1660             for (DpnInterfaces dpnIf : dpnInterfaces) {
1661                 BigInteger dpnIfDpId = dpnIf.getDpId();
1662                 if (dpnIfDpId.equals(srcDpId)) {
1663                     cnt++;
1664                 } else if (dpnIfDpId.equals(dstDpId)) {
1665                     cnt++;
1666                     dstDpnIf = dpnIf;
1667                 }
1668             }
1669             if (cnt == 2) {
1670                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1671                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1672                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1673                 dataStoreCoordinator.enqueueJob(elanName, () -> {
1674                     // update Remote BC Group
1675                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1676                     List<ListenableFuture<Void>> elanFutures = new ArrayList<>();
1677                     try {
1678                         setupElanBroadcastGroups(elanInfo, srcDpId);
1679                     } catch (RuntimeException e) {
1680                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1681                         ElanUtils.addToListenableFutureIfTxException(e, elanFutures);
1682                     }
1683                     Set<String> interfaceLists = new HashSet<>();
1684                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1685                     for (String ifName : interfaceLists) {
1686                         dataStoreCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1687                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1688                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1689                             List<ListenableFuture<Void>> elanInterfacefutures = new ArrayList<>();
1690                             if (isOperational(interfaceInfo)) {
1691                                 installDMacAddressTables(elanInfo, interfaceInfo, srcDpId, elanInterfacefutures);
1692                             }
1693                             return elanInterfacefutures;
1694                         }, ElanConstants.JOB_MAX_RETRIES);
1695                     }
1696                     return elanFutures;
1697                 }, ElanConstants.JOB_MAX_RETRIES);
1698             }
1699
1700         }
1701     }
1702
1703     /**
1704      * Handle external tunnel state event.
1705      *
1706      * @param externalTunnel
1707      *            the external tunnel
1708      * @param intrf
1709      *            the interface
1710      * @throws ElanException in case of issues creating the flow objects
1711      */
1712     public void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) throws ElanException {
1713         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1714             return;
1715         }
1716         // dpId/externalNodeId will be available either in source or destination
1717         // based on the tunnel end point
1718         BigInteger dpId = null;
1719         NodeId externalNodeId = null;
1720         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1721             dpId = new BigInteger(externalTunnel.getSourceDevice());
1722             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1723         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1724             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1725             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1726         }
1727         if (dpId == null || externalNodeId == null) {
1728             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1729             return;
1730         }
1731
1732         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1733         if (dpnInterfaceLists == null) {
1734             return;
1735         }
1736         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1737         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1738             String elanName = elanDpns.getElanInstanceName();
1739             ElanInstance elanInfo = ElanUtils.getElanInstanceByName(broker, elanName);
1740
1741             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1742             if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1743                     || dpnInterfaces.getInterfaces().isEmpty()) {
1744                 continue;
1745             }
1746             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1747
1748             setupElanBroadcastGroups(elanInfo, dpId);
1749             // install L2gwDevices local macs in dpn.
1750             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1751             // Install dpn macs on external device
1752             elanL2GatewayUtils.installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1753                     externalNodeId);
1754         }
1755         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1756     }
1757
1758     /**
1759      * Validate external tunnel state event.
1760      *
1761      * @param externalTunnel
1762      *            the external tunnel
1763      * @param intrf
1764      *            the intrf
1765      * @return true, if successful
1766      */
1767     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1768         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1769             String srcDevice = externalTunnel.getDestinationDevice();
1770             String destDevice = externalTunnel.getSourceDevice();
1771             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1772                     LogicalDatastoreType.CONFIGURATION);
1773             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1774                     otherEndPointExtTunnel);
1775             if (otherEndPointExtTunnel != null) {
1776                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1777                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1778                 if (otherEndPointInterfaceOperational) {
1779                     return true;
1780                 } else {
1781                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1782                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1783                 }
1784             }
1785         }
1786         return false;
1787     }
1788
1789     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1790         List<MatchInfo> mkMatches = new ArrayList<>();
1791         // Matching metadata
1792         mkMatches.add(
1793                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1794         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1795         return mkMatches;
1796     }
1797
1798     public void updateRemoteBroadcastGroupForAllElanDpns(ElanInstance elanInfo) {
1799         List<DpnInterfaces> dpns = elanUtils.getInvolvedDpnsInElan(elanInfo.getElanInstanceName());
1800         if (dpns == null) {
1801             return;
1802         }
1803         for (DpnInterfaces dpn : dpns) {
1804             setupElanBroadcastGroups(elanInfo, dpn.getDpId());
1805         }
1806     }
1807
1808     public List<Bucket> getRemoteBCGroupBucketsOfElanL2GwDevices(ElanInstance elanInfo, BigInteger dpnId,
1809             int bucketId) {
1810         List<Bucket> listBucketInfo = new ArrayList<>();
1811         ConcurrentMap<String, L2GatewayDevice> map = ElanL2GwCacheUtils
1812                 .getInvolvedL2GwDevices(elanInfo.getElanInstanceName());
1813         for (L2GatewayDevice device : map.values()) {
1814             String interfaceName = elanL2GatewayUtils.getExternalTunnelInterfaceName(String.valueOf(dpnId),
1815                     device.getHwvtepNodeId());
1816             if (interfaceName == null) {
1817                 continue;
1818             }
1819             List<Action> listActionInfo = elanUtils.buildTunnelItmEgressActions(interfaceName,
1820                     ElanUtils.getVxlanSegmentationId(elanInfo));
1821             listBucketInfo.add(MDSALUtil.buildBucket(listActionInfo, MDSALUtil.GROUP_WEIGHT, bucketId,
1822                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1823             bucketId++;
1824         }
1825         return listBucketInfo;
1826     }
1827
1828     public List<Bucket> getRemoteBCGroupBucketsOfElanExternalTeps(ElanInstance elanInfo, BigInteger dpnId,
1829                                                                   int bucketId) {
1830         List<Bucket> listBucketInfo = new ArrayList<>();
1831         List<ExternalTeps> teps = elanInfo.getExternalTeps();
1832         if (teps == null || teps.isEmpty()) {
1833             return listBucketInfo;
1834         }
1835         for (ExternalTeps tep : teps) {
1836             String interfaceName = elanL2GatewayUtils.getExternalTunnelInterfaceName(String.valueOf(dpnId),
1837                     tep.getTepIp().toString());
1838             if (interfaceName == null) {
1839                 LOG.error("Could not get interface name to ext tunnel {} {}", dpnId, tep.getTepIp());
1840                 continue;
1841             }
1842             List<Action> listActionInfo = elanUtils.buildTunnelItmEgressActions(interfaceName,
1843                     elanInfo.getSegmentationId());
1844             listBucketInfo.add(MDSALUtil.buildBucket(listActionInfo, MDSALUtil.GROUP_WEIGHT, bucketId,
1845                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1846             bucketId++;
1847         }
1848         return listBucketInfo;
1849     }
1850
1851     @Override
1852     protected ElanInterfaceManager getDataTreeChangeListener() {
1853         return this;
1854     }
1855
1856     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1857                                              BigInteger dpId) {
1858         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1859         setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId);
1860         try {
1861             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1862         } catch (InterruptedException e) {
1863             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1864         }
1865     }
1866 }