elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.base.Preconditions;
  15 import com.google.common.collect.Lists;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.HashSet;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Objects;
  25 import java.util.Queue;
  26 import java.util.Set;
  27 import java.util.concurrent.ConcurrentHashMap;
  28 import java.util.concurrent.ConcurrentLinkedQueue;
  29 import javax.annotation.PostConstruct;
  30 import javax.inject.Inject;
  31 import javax.inject.Singleton;
  32 import org.apache.commons.lang3.StringUtils;
  33 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  34 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  38 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  39 import org.opendaylight.genius.infra.Datastore.Configuration;
  40 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  42 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  43 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  44 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  45 import org.opendaylight.genius.itm.globals.ITMConstants;
  46 import org.opendaylight.genius.mdsalutil.FlowEntity;
  47 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  48 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  49 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  50 import org.opendaylight.genius.mdsalutil.MatchInfo;
  51 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  52 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  53 import org.opendaylight.genius.mdsalutil.NwConstants;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  59 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  60 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  61 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  62 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  63 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  64 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  65 import org.opendaylight.genius.utils.ServiceIndex;
  66 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  67 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  68 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  69 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  70 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  71 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  72 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  73 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  74 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  75 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  76 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  77 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  78 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  79 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  80 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  81 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  82 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  83 import org.opendaylight.serviceutils.srm.RecoverableListener;
  84 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  85 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  86 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 128 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 129 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 130 import org.slf4j.Logger;
 131 import org.slf4j.LoggerFactory;
 132
 133 /**
 134  * Class in charge of handling creations, modifications and removals of
 135  * ElanInterfaces.
 136  *
 137  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 138  */
 139 @Singleton
 140 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 141         implements RecoverableListener {
 142     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 143     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 144     private static final boolean SH_FLAG_SET = true;
 145     private static final boolean SH_FLAG_UNSET = false;
 146
 147     private final DataBroker broker;
 148     private final ManagedNewTransactionRunner txRunner;
 149     private final IMdsalApiManager mdsalManager;
 150     private final IInterfaceManager interfaceManager;
 151     private final IdManagerService idManager;
 152     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 153     private final INeutronVpnManager neutronVpnManager;
 154     private final ElanItmUtils elanItmUtils;
 155     private final ElanEtreeUtils elanEtreeUtils;
 156     private final ElanL2GatewayUtils elanL2GatewayUtils;
 157     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 158     private final ElanUtils elanUtils;
 159     private final JobCoordinator jobCoordinator;
 160     private final ElanInstanceCache elanInstanceCache;
 161     private final ElanInterfaceCache elanInterfaceCache;
 162
 163     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 164         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 165
 166     @Inject
 167     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 168                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 169                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 170                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 171                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 172                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 173                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 174                                 final ElanInstanceCache elanInstanceCache,
 175                                 final ElanInterfaceCache elanInterfaceCache,
 176                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 177                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 178         super(ElanInterface.class, ElanInterfaceManager.class);
 179         this.broker = dataBroker;
 180         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 181         this.idManager = managerService;
 182         this.mdsalManager = mdsalApiManager;
 183         this.interfaceManager = interfaceManager;
 184         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 185         this.neutronVpnManager = neutronVpnManager;
 186         this.elanItmUtils = elanItmUtils;
 187         this.elanEtreeUtils = elanEtreeUtils;
 188         this.elanL2GatewayUtils = elanL2GatewayUtils;
 189         this.elanUtils = elanUtils;
 190         this.jobCoordinator = jobCoordinator;
 191         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 192         this.elanInstanceCache = elanInstanceCache;
 193         this.elanInterfaceCache = elanInterfaceCache;
 194         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 195     }
 196
 197     @Override
 198     @PostConstruct
 199     public void init() {
 200         registerListener();
 201     }
 202
 203     @Override
 204     public void registerListener() {
 205         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 206     }
 207
 208     @Override
 209     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 210         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 211     }
 212
 213     @Override
 214     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 215         String interfaceName = del.getName();
 216         String elanInstanceName = del.getElanInstanceName();
 217         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 218         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 219             elanInterfaces.remove(del);
 220             if (elanInterfaces.isEmpty()) {
 221                 unProcessedElanInterfaces.remove(elanInstanceName);
 222             }
 223         }
 224         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 225         /*
 226          * Handling in case the elan instance is deleted.If the Elan instance is
 227          * deleted, there is no need to explicitly delete the elan interfaces
 228          */
 229         if (elanInfo == null) {
 230             return;
 231         }
 232         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 233         if (interfaceInfo == null && elanInfo.isExternal()) {
 234             // In deleting external network, the underlying ietf Inteface might have been removed
 235             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 236             // from Operational DS instead
 237             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 238         }
 239         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 240                 interfaceName, interfaceInfo, this);
 241         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 242     }
 243
 244     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 245     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 246             InterfaceInfo interfaceInfo) {
 247         String elanName = elanInfo.getElanInstanceName();
 248         boolean isLastElanInterface = false;
 249         boolean isLastInterfaceOnDpn = false;
 250         BigInteger dpId = null;
 251         long elanTag = elanInfo.getElanTag();
 252         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 253         WriteTransaction interfaceTx = broker.newWriteOnlyTransaction();
 254         Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 255         if (elanState == null) {
 256             interfaceTx.cancel();
 257             return Collections.emptyList();
 258         }
 259         WriteTransaction flowTx = broker.newWriteOnlyTransaction();
 260         List<String> elanInterfaces = elanState.getElanInterfaces();
 261         if (elanInterfaces.isEmpty()) {
 262             isLastElanInterface = true;
 263         }
 264         if (interfaceInfo != null) {
 265             dpId = interfaceInfo.getDpId();
 266             DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, dpId,
 267                     interfaceName, elanTag, interfaceTx);
 268             /*
 269              * If there are not elan ports, remove the unknown dmac, terminating
 270              * service table flows, remote/local bc group
 271              */
 272             if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 273                     || dpnInterfaces.getInterfaces().isEmpty()) {
 274                 // No more Elan Interfaces in this DPN
 275                 LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(), dpId);
 276                 if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 277                     removeDefaultTermFlow(dpId, elanInfo.getElanTag());
 278                 }
 279                 removeUnknownDmacFlow(dpId, elanInfo, flowTx, elanInfo.getElanTag());
 280                 removeEtreeUnknownDmacFlow(dpId, elanInfo, flowTx);
 281                 removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 282                 removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 283                 removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 284                 if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 285                     if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 286                         elanUtils.removeTerminatingServiceAction(dpId,
 287                                 ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 288                     }
 289                     unsetExternalTunnelTable(dpId, elanInfo);
 290                 }
 291                 isLastInterfaceOnDpn = true;
 292             } else {
 293                 setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 294             }
 295         }
 296
 297         List<ListenableFuture<Void>> futures = new ArrayList<>();
 298         futures.add(ElanUtils.waitForTransactionToComplete(interfaceTx));
 299         futures.add(ElanUtils.waitForTransactionToComplete(flowTx));
 300
 301         if (isLastInterfaceOnDpn && dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 302             setElanAndEtreeBCGrouponOtherDpns(elanInfo, dpId);
 303         }
 304         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 305                 interfaceName, elanInfo, interfaceInfo, this, isLastElanInterface);
 306         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 307                 ElanConstants.JOB_MAX_RETRIES);
 308
 309         return futures;
 310     }
 311
 312     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 313             WriteTransaction deleteFlowGroupTx) {
 314         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 315         if (etreeLeafTag != null) {
 316             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 317             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 318         }
 319     }
 320
 321     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 322             WriteTransaction deleteFlowGroupTx) {
 323         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 324         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 325     }
 326
 327     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 328             WriteTransaction deleteFlowGroupTx) {
 329         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 330         if (etreeInstance != null) {
 331             BigInteger dpnId = interfaceInfo.getDpId();
 332             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 333             List<Bucket> listBuckets = new ArrayList<>();
 334             int bucketId = 0;
 335             listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
 336             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 337                     MDSALUtil.buildBucketLists(listBuckets));
 338             LOG.trace("deleted the localBroadCast Group:{}", group);
 339             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 340         }
 341     }
 342
 343     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 344             WriteTransaction deleteFlowGroupTx) {
 345         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 346         if (etreeInstance != null) {
 347             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 348             int bucketId = 0;
 349             int actionKey = 0;
 350             List<Bucket> listBuckets = new ArrayList<>();
 351             List<Action> listAction = new ArrayList<>();
 352             listAction.add(new ActionGroup(ElanUtils.getEtreeLeafLocalBCGId(etreeTag)).buildAction(++actionKey));
 353             listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
 354                     MDSALUtil.WATCH_GROUP));
 355             bucketId++;
 356             listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, etreeTag));
 357             BigInteger dpnId = interfaceInfo.getDpId();
 358             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 359             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 360                     MDSALUtil.buildBucketLists(listBuckets));
 361             LOG.trace("deleting the remoteBroadCast group:{}", group);
 362             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 363         }
 364     }
 365
 366     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName, WriteTransaction tx) {
 367         String elanName = elanInfo.getElanInstanceName();
 368         Elan elanState = ElanUtils.getElanByName(broker, elanName);
 369         if (elanState == null) {
 370             return elanState;
 371         }
 372         List<String> elanInterfaces = elanState.getElanInterfaces();
 373         boolean isRemoved = elanInterfaces.remove(interfaceName);
 374         if (!isRemoved) {
 375             return elanState;
 376         }
 377
 378         if (elanInterfaces.isEmpty()) {
 379             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName));
 380             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanMacTableOperationalDataPath(elanName));
 381             tx.delete(LogicalDatastoreType.OPERATIONAL,
 382                     ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 383         } else {
 384             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 385                     .withKey(new ElanKey(elanName)).build();
 386             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName),
 387                     updateElanState);
 388         }
 389         return elanState;
 390     }
 391
 392     private void deleteElanInterfaceFromConfigDS(String interfaceName, WriteTransaction tx) {
 393         // removing the ElanInterface from the config data_store if interface is
 394         // not present in Interface config DS
 395         if (interfaceManager.getInterfaceInfoFromConfigDataStore(interfaceName) == null
 396                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 397             tx.delete(LogicalDatastoreType.CONFIGURATION,
 398                     ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 399         }
 400     }
 401
 402     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 403             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 404         String elanName = elanInfo.getElanInstanceName();
 405         List<ListenableFuture<Void>> futures = new ArrayList<>();
 406         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 407             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(interfaceTx -> {
 408                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 409                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 410                 Optional<ElanInterfaceMac> existingElanInterfaceMac =
 411                         interfaceTx.read(LogicalDatastoreType.OPERATIONAL, elanInterfaceId).checkedGet();
 412                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 413                 if (interfaceInfo != null) {
 414                     if (existingElanInterfaceMac.isPresent()) {
 415                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 416                         if (existingMacEntries != null) {
 417                             List<PhysAddress> macAddresses = new ArrayList<>();
 418                             for (MacEntry macEntry : existingMacEntries) {
 419                                 PhysAddress macAddress = macEntry.getMacAddress();
 420                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 421                                         macAddress.getValue(), interfaceName);
 422                                 Optional<MacEntry> macEntryOptional =
 423                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 424                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 425                                     interfaceTx.delete(LogicalDatastoreType.OPERATIONAL,
 426                                             ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 427                                 }
 428                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 429                                 macAddresses.add(macAddress);
 430                             }
 431
 432                             // Removing all those MACs from External Devices belonging
 433                             // to this ELAN
 434                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 435                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 436                             }
 437                         }
 438                     }
 439                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 440                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 441                 } else if (existingElanInterfaceMac.isPresent()) {
 442                     // Interface does not exist in ConfigDS, so lets remove everything
 443                     // about that interface related to Elan
 444                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 445                     if (macEntries != null) {
 446                         for (MacEntry macEntry : macEntries) {
 447                             PhysAddress macAddress = macEntry.getMacAddress();
 448                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 449                                 interfaceTx.delete(LogicalDatastoreType.OPERATIONAL,
 450                                         ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 451                             }
 452                         }
 453                     }
 454                 }
 455                 if (existingElanInterfaceMac.isPresent()) {
 456                     interfaceTx.delete(LogicalDatastoreType.OPERATIONAL, elanInterfaceId);
 457                 }
 458                 unbindService(interfaceName, interfaceTx);
 459                 deleteElanInterfaceFromConfigDS(interfaceName, interfaceTx);
 460             }));
 461         }));
 462         return futures;
 463     }
 464
 465     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 466                                                                          String interfaceName, long elanTag,
 467                                                                          WriteTransaction tx) {
 468         synchronized (elanName.intern()) {
 469
 470             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 471             if (dpnInterfaces != null) {
 472                 List<String> interfaceLists = dpnInterfaces.getInterfaces();
 473                 if (interfaceLists != null) {
 474                     interfaceLists.remove(interfaceName);
 475                 }
 476
 477                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 478                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 479                     deleteElanDpnInterface(elanName, dpId, tx);
 480                 } else {
 481                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 482                 }
 483             }
 484             return dpnInterfaces;
 485         }
 486     }
 487
 488     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 489         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 490         for (DpnInterfaces dpnInterface : dpnInterfaces) {
 491             BigInteger currentDpId = dpnInterface.getDpId();
 492             if (!currentDpId.equals(dpId)) {
 493                 for (String elanInterface : dpnInterface.getInterfaces()) {
 494                     ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 495                     if (macs == null || macs.getMacEntry() == null) {
 496                         continue;
 497                     }
 498                     for (MacEntry mac : macs.getMacEntry()) {
 499                         removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 500                         removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 501                     }
 502                 }
 503             }
 504         }
 505     }
 506
 507     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 508         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 509         if (etreeLeafTag != null) {
 510             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac);
 511         }
 512     }
 513
 514     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 515         mdsalManager
 516                 .removeFlow(dpId,
 517                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 518                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 519                                         mac.getMacAddress().getValue(), elanTag)));
 520     }
 521
 522     /*
 523     * Possible Scenarios for update
 524     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 525         then all {1,2,3,4} should be removed
 526
 527         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 528         then all {1,2,3,4} should be added
 529
 530         c. if orig = {1,2,3,4} updated={2,3,4}
 531         then 1 should be removed
 532
 533         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 534         then we should just add 5
 535
 536         e. if orig = {1,2,3,4} updated={2,3,4,5}
 537         then 1 should be removed , 5 should be added
 538     * */
 539     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 540     @Override
 541     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 542         // updating the static-Mac Entries for the existing elanInterface
 543         String elanName = update.getElanInstanceName();
 544         String interfaceName = update.getName();
 545
 546         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 547         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 548         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 549         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 550
 551         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 552                 deletedEntry.getMacAddress()));
 553
 554         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 555         * Hence add the macentries for the same.*/
 556         for (StaticMacEntries staticMacEntry : updatedEntries) {
 557             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 558                     staticMacEntry.getMacAddress());
 559             Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 560                     LogicalDatastoreType.OPERATIONAL, macEntryIdentifier);
 561             WriteTransaction tx = broker.newWriteOnlyTransaction();
 562             if (existingMacEntry.isPresent()) {
 563                 elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 564                         elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 565                         tx);
 566             } else {
 567                 elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 568                         elanName, interfaceName, staticMacEntry, tx);
 569             }
 570             ListenableFutures.addErrorLogging(ElanUtils.waitForTransactionToComplete(tx), LOG,
 571                     "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 572         }
 573     }
 574
 575     @Override
 576     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 577         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 578             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 579             String interfaceName = elanInterfaceAdded.getName();
 580             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 581             if (interfaceInfo == null) {
 582                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 583                 return;
 584             }
 585             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 586
 587             if (elanInstance == null) {
 588                 elanInstance = new ElanInstanceBuilder().setElanInstanceName(elanInstanceName)
 589                         .setDescription(elanInterfaceAdded.getDescription()).build();
 590                 // Add the ElanInstance in the Configuration data-store
 591                 List<String> elanInterfaces = new ArrayList<>();
 592                 elanInterfaces.add(interfaceName);
 593                 elanInstance = ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, tx);
 594             }
 595
 596             Long elanTag = elanInstance.getElanTag();
 597             // If elan tag is not updated, then put the elan interface into
 598             // unprocessed entry map and entry. Let entries
 599             // in this map get processed during ELAN update DCN.
 600             if (elanTag == null || elanTag == 0L) {
 601                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 602                 if (elanInterfaces == null) {
 603                     elanInterfaces = new ConcurrentLinkedQueue<>();
 604                 }
 605                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 606                     elanInterfaces.add(elanInterfaceAdded);
 607                 }
 608                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 609                 return;
 610             }
 611             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 612                     interfaceInfo, elanInstance, this);
 613             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 614         }), LOG, "Error procedding added ELAN interface");
 615     }
 616
 617     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 618         List<ListenableFuture<Void>> futures = new ArrayList<>();
 619         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 620         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 621             return futures;
 622         }
 623         for (ElanInterface elanInterface : elanInterfaces) {
 624             String interfaceName = elanInterface.getName();
 625             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 626             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 627         }
 628         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 629         return futures;
 630     }
 631
 632     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 633             WriteTransaction writeFlowGroupTx) {
 634         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 635                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 636         List<DpnInterfaces> dpnInterfaceLists = null;
 637         if (elanDpnInterfacesList != null) {
 638             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 639         }
 640         if (dpnInterfaceLists == null) {
 641             dpnInterfaceLists = new ArrayList<>();
 642         }
 643         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 644             BigInteger dstDpId = interfaceInfo.getDpId();
 645             if (dpnInterfaces.getDpId().equals(dstDpId)) {
 646                 continue;
 647             }
 648             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 649             for (String remoteIf : remoteElanInterfaces) {
 650                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 651                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 652                 if (elanIfMac == null || remoteInterface == null) {
 653                     continue;
 654                 }
 655                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 656                 if (remoteMacEntries != null) {
 657                     for (MacEntry macEntry : remoteMacEntries) {
 658                         String macAddress = macEntry.getMacAddress().getValue();
 659                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 660                                 dstDpId, elanInstance.getElanInstanceName());
 661                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 662                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 663                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 664                     }
 665                 }
 666             }
 667         }
 668     }
 669
 670     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 671     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 672             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 673         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 674         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 675         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 676
 677         String interfaceName = elanInterface.getName();
 678         String elanInstanceName = elanInterface.getElanInstanceName();
 679
 680         Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 681         WriteTransaction tx = broker.newWriteOnlyTransaction();
 682         if (elanInfo == null) {
 683             List<String> elanInterfaces = new ArrayList<>();
 684             elanInterfaces.add(interfaceName);
 685             ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, tx);
 686         } else {
 687             createElanStateList(elanInstanceName, interfaceName, tx);
 688         }
 689         boolean isFirstInterfaceInDpn = false;
 690         // Specific actions to the DPN where the ElanInterface has been added,
 691         // for example, programming the
 692         // External tunnel table if needed or adding the ElanInterface to the
 693         // DpnInterfaces in the operational DS.
 694         BigInteger dpId = interfaceInfo.getDpId();
 695         DpnInterfaces dpnInterfaces = null;
 696         if (dpId != null && !dpId.equals(ElanConstants.INVALID_DPN)) {
 697             synchronized (elanInstanceName.intern()) {
 698                 InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 699                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
 700                 Optional<DpnInterfaces> existingElanDpnInterfaces = ElanUtils.read(broker,
 701                         LogicalDatastoreType.OPERATIONAL, elanDpnInterfaces);
 702                 if (ElanUtils.isVlan(elanInstance)) {
 703                     isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 704                             elanInstanceName, existingElanDpnInterfaces);
 705                 } else {
 706                     isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 707                 }
 708                 if (isFirstInterfaceInDpn) {
 709                     // ELAN's 1st ElanInterface added to this DPN
 710                     if (!existingElanDpnInterfaces.isPresent()) {
 711                         dpnInterfaces = createElanInterfacesList(elanInstanceName, interfaceName, dpId, tx);
 712                     } else {
 713                         List<String> elanInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 714                         elanInterfaces.add(interfaceName);
 715                         dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, dpId,
 716                                 elanInterfaces, tx);
 717                     }
 718                     // The 1st ElanInterface in a DPN must program the Ext Tunnel
 719                     // table, but only if Elan has VNI
 720                     if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 721                         setExternalTunnelTable(dpId, elanInstance);
 722                     }
 723                     elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 724                 } else {
 725                     List<String> elanInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 726                     elanInterfaces.add(interfaceName);
 727                     if (elanInterfaces.size() == 1) { // 1st dpn interface
 728                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 729                     }
 730                     dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, dpId, elanInterfaces, tx);
 731                 }
 732             }
 733         }
 734
 735         // add code to install Local/Remote BC group, unknow DMAC entry,
 736         // terminating service table flow entry
 737         // call bindservice of interfacemanager to create ingress table flow
 738         // enty.
 739         // Add interface to the ElanInterfaceForwardingEntires Container
 740         createElanInterfaceTablesList(interfaceName, tx);
 741         List<ListenableFuture<Void>> futures = new ArrayList<>();
 742         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 743         installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, dpnInterfaces, isFirstInterfaceInDpn);
 744
 745         // add the vlan provider interface to remote BC group for the elan
 746         // for internal vlan networks
 747         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 748             if (interfaceManager.isExternalInterface(interfaceName)) {
 749                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 750                 handleExternalInterfaceEvent(elanInstance, dpnInterfaces, dpId);
 751             }
 752         }
 753
 754         if (isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 755             //update the remote-DPNs remoteBC group entry with Tunnels
 756             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance, dpId);
 757             setElanAndEtreeBCGrouponOtherDpns(elanInstance, dpId);
 758         }
 759
 760         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 761         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 762                 elanInterface, interfaceInfo, elanInstance, isFirstInterfaceInDpn, this);
 763         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 764         return futures;
 765     }
 766
 767     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 768     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 769             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 770         String elanInstanceName = elanInstance.getElanInstanceName();
 771         String interfaceName = elanInterface.getName();
 772         WriteTransaction tx = broker.newWriteOnlyTransaction();
 773         BigInteger dpId = interfaceInfo.getDpId();
 774         WriteTransaction writeFlowGroupTx = broker.newWriteOnlyTransaction();
 775         installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 776                 isFirstInterfaceInDpn, tx, writeFlowGroupTx);
 777
 778         List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 779         List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 780
 781         boolean isInterfaceOperational = isOperational(interfaceInfo);
 782         if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 783             for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 784                 InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 785                         staticMacEntry.getMacAddress());
 786                 Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 787                         LogicalDatastoreType.OPERATIONAL, macId);
 788                 if (existingMacEntry.isPresent()) {
 789                     elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 790                             elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 791                             existingMacEntry.get(), tx);
 792                 } else {
 793                     elanForwardingEntriesHandler
 794                             .addElanInterfaceForwardingTableList(elanInstanceName, interfaceName, staticMacEntry, tx);
 795                 }
 796
 797                 if (isInterfaceOperational) {
 798                     // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 799                     // DPNs
 800                     String macAddress = staticMacEntry.getMacAddress().getValue();
 801                     LOG.info("programming smac and dmacs for {} on source and other DPNs for elan {} and interface {}",
 802                             macAddress, elanInstanceName, interfaceName);
 803                     elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 804                             staticMacEntry.getMacAddress().getValue(), true, writeFlowGroupTx);
 805                 }
 806             }
 807
 808             if (isInterfaceOperational) {
 809                 // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 810                 // on purpose.
 811                 for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 812                     staticMacAddresses.add(staticMacEntry.getMacAddress());
 813                 }
 814                 elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 815                         staticMacAddresses);
 816             }
 817         }
 818         List<ListenableFuture<Void>> futures = new ArrayList<>();
 819         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 820         futures.add(ElanUtils.waitForTransactionToComplete(writeFlowGroupTx));
 821         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 822             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 823             try {
 824                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 825                 if (neutronPort != null) {
 826                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 827                 }
 828             } catch (IllegalArgumentException ex) {
 829                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 830             }
 831         }
 832         return futures;
 833     }
 834
 835     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 836             PhysAddress physAddress) {
 837         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 838         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 839         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 840                 LogicalDatastoreType.OPERATIONAL, macId);
 841
 842         if (!existingMacEntry.isPresent()) {
 843             return;
 844         }
 845
 846         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 847                 .withKey(new MacEntryKey(physAddress)).build();
 848         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 849                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 850     }
 851
 852     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 853             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 854         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 855         if (!existingElanDpnInterfaces.isPresent()) {
 856             return true;
 857         }
 858         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 859             return false;
 860         }
 861         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 862         int dummyInterfaceCount =  0;
 863         if (dpnInterfaces.getInterfaces().contains(routerPortUuid)) {
 864             dummyInterfaceCount++;
 865         }
 866         if (dpnInterfaces.getInterfaces().contains(elanInstanceName)) {
 867             dummyInterfaceCount++;
 868         }
 869         if (dpnInterfaces.getInterfaces().size() - dummyInterfaceCount == 0) {
 870             return true;
 871         }
 872         return false;
 873     }
 874
 875     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 876         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 877                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 878     }
 879
 880     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 881             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, WriteTransaction tx,
 882             WriteTransaction writeFlowGroupTx) {
 883         if (!isOperational(interfaceInfo)) {
 884             return;
 885         }
 886         BigInteger dpId = interfaceInfo.getDpId();
 887         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 888             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, writeFlowGroupTx);
 889         }
 890         setupFilterEqualsTable(elanInstance, interfaceInfo, writeFlowGroupTx);
 891         if (isFirstInterfaceInDpn) {
 892             // Terminating Service , UnknownDMAC Table.
 893             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 894             // for ELAN Instance is VxLAN
 895             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 896                 setupTerminateServiceTable(elanInstance, dpId, writeFlowGroupTx);
 897             }
 898             setupUnknownDMacTable(elanInstance, dpId, writeFlowGroupTx);
 899             /*
 900              * Install remote DMAC flow. This is required since this DPN is
 901              * added later to the elan instance and remote DMACs of other
 902              * interfaces in this elan instance are not present in the current
 903              * dpn.
 904              */
 905             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 906                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 907                         elanInstance.getElanInstanceName());
 908                 programRemoteDmacFlow(elanInstance, interfaceInfo, writeFlowGroupTx);
 909             }
 910         }
 911         // bind the Elan service to the Interface
 912         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), tx);
 913     }
 914
 915     public void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 916             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn) {
 917         if (!isOperational(interfaceInfo)) {
 918             return;
 919         }
 920         // LocalBroadcast Group creation with elan-Interfaces
 921         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 922         if (isFirstInterfaceInDpn) {
 923             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 924             BigInteger dpId = interfaceInfo.getDpId();
 925             // RemoteBroadcast Group creation
 926             try {
 927                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 928             } catch (InterruptedException e1) {
 929                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 930             }
 931             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId);
 932             try {
 933                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 934             } catch (InterruptedException e1) {
 935                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 936             }
 937         }
 938     }
 939
 940     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 941             WriteTransaction writeFlowGroupTx) {
 942         int ifTag = interfaceInfo.getInterfaceTag();
 943         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 944                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 945                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 946                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 947                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 948
 949         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flow, writeFlowGroupTx);
 950
 951         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 952                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 953                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 954                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 955
 956         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flowEntry, writeFlowGroupTx);
 957     }
 958
 959     // TODO skitt Fix the exception handling here
 960     @SuppressWarnings("checkstyle:IllegalCatch")
 961     @SuppressFBWarnings("REC_CATCH_EXCEPTION")
 962     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 963             TypedReadWriteTransaction<Configuration> flowTx) {
 964         try {
 965             int ifTag = interfaceInfo.getInterfaceTag();
 966             Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 967                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
 968
 969             mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
 970
 971             Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 972                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 973                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 974                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 975
 976             mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
 977         } catch (Exception e) {
 978             LOG.error("Error removing flow", e);
 979             throw new RuntimeException("Error removing flow", e);
 980         }
 981     }
 982
 983     private List<Bucket> getRemoteBCGroupBucketInfos(ElanInstance elanInfo, int bucketKeyStart,
 984                                                      InterfaceInfo interfaceInfo, long elanTag) {
 985         return elanL2GatewayMulticastUtils.getRemoteBCGroupBuckets(elanInfo, null, interfaceInfo.getDpId(),
 986                 bucketKeyStart, elanTag);
 987     }
 988
 989     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId) {
 990         int elanTag = elanInfo.getElanTag().intValue();
 991         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
 992         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId);
 993         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 994         if (etreeInstance != null) {
 995             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
 996             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
 997             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId);
 998         }
 999     }
1000
1001     @SuppressWarnings("checkstyle:IllegalCatch")
1002     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId) {
1003         int bucketId = 0;
1004         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
1005         if (elanDpns != null) {
1006             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1007             for (DpnInterfaces dpnInterface : dpnInterfaces) {
1008                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
1009                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
1010                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
1011                     List<Action> listAction = new ArrayList<>();
1012                     int actionKey = 0;
1013                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1014                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
1015                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1016                     bucketId++;
1017                     for (DpnInterfaces otherFes : dpnInterfaces) {
1018                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
1019                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
1020                             && !otherFes.getInterfaces().isEmpty()) {
1021                             try {
1022                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
1023                                         dpnInterface.getDpId(), otherFes.getDpId(),
1024                                         elanUtils.isOpenstackVniSemanticsEnforced()
1025                                                 ? ElanUtils.getVxlanSegmentationId(elanInfo) : elanTag);
1026                                 if (!remoteListActionInfo.isEmpty()) {
1027                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
1028                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1029                                     bucketId++;
1030                                 }
1031                             } catch (Exception ex) {
1032                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1033                                         + "Logical Group Interface not found between source Dpn - {}, "
1034                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1035                                 return;
1036                             }
1037                         }
1038                     }
1039                     List<Bucket> elanL2GwDevicesBuckets = elanL2GatewayMulticastUtils
1040                             .getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnInterface.getDpId(), bucketId);
1041                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1042
1043                     if (remoteListBucketInfo.isEmpty()) {
1044                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1045                         continue;
1046                     }
1047                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1048                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1049                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1050                     mdsalManager.syncInstallGroup(dpnInterface.getDpId(), group);
1051                 }
1052             }
1053             try {
1054                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1055             } catch (InterruptedException e1) {
1056                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1057             }
1058         }
1059     }
1060
1061     /**
1062      * Returns the bucket info with the given interface as the only bucket.
1063      */
1064     private Bucket getLocalBCGroupBucketInfo(InterfaceInfo interfaceInfo, int bucketIdStart) {
1065         return MDSALUtil.buildBucket(getInterfacePortActions(interfaceInfo), MDSALUtil.GROUP_WEIGHT, bucketIdStart,
1066                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
1067     }
1068
1069     private List<MatchInfo> buildMatchesForVni(Long vni) {
1070         List<MatchInfo> mkMatches = new ArrayList<>();
1071         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1072         mkMatches.add(match);
1073         return mkMatches;
1074     }
1075
1076     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1077         List<InstructionInfo> mkInstructions = new ArrayList<>();
1078         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1079         return mkInstructions;
1080     }
1081
1082     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1083         List<MatchInfo> mkMatches = new ArrayList<>();
1084         // Matching metadata
1085         mkMatches.add(new MatchMetadata(
1086                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1087         return mkMatches;
1088     }
1089
1090     /**
1091      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1092      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1093      *
1094      * @param elanTag
1095      *            elanTag to be written in metadata when flow is selected
1096      * @return the instructions ready to be installed in a flow
1097      */
1098     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1099         List<InstructionInfo> mkInstructions = new ArrayList<>();
1100         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1101                 .getElanMetadataMask()));
1102         /* applicable for EXTERNAL_TUNNEL_TABLE only
1103         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1104         */
1105         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1106         return mkInstructions;
1107     }
1108
1109     // Install DMAC entry on dst DPN
1110     @SuppressWarnings("checkstyle:ForbidCertainMethod")
1111     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1112             BigInteger dstDpId) {
1113         String interfaceName = interfaceInfo.getInterfaceName();
1114         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1115         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1116             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1117             WriteTransaction writeFlowTx = broker.newWriteOnlyTransaction();
1118             for (MacEntry macEntry : macEntries) {
1119                 String macAddress = macEntry.getMacAddress().getValue();
1120                 LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress, interfaceName);
1121                 synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1122                         interfaceInfo.getDpId())) {
1123                     LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation", macAddress);
1124                     elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress,
1125                             writeFlowTx);
1126                 }
1127             }
1128             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(writeFlowTx));
1129         }
1130         return Collections.emptyList();
1131     }
1132
1133     private void createDropBucket(List<Bucket> listBucket) {
1134         List<Action> actionsInfos = new ArrayList<>();
1135         actionsInfos.add(new ActionDrop().buildAction());
1136         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1137                 MDSALUtil.WATCH_GROUP);
1138         listBucket.add(dropBucket);
1139     }
1140
1141     public void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1142             InterfaceInfo interfaceInfo) {
1143         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1144         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1145     }
1146
1147     public void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1148             InterfaceInfo interfaceInfo) {
1149         List<Bucket> listBucket = new ArrayList<>();
1150         int bucketId = 0;
1151         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1152
1153         List<String> interfaces = new ArrayList<>();
1154         if (newDpnInterface != null) {
1155             interfaces = newDpnInterface.getInterfaces();
1156         }
1157         for (String ifName : interfaces) {
1158             // In case if there is a InterfacePort in the cache which is not in
1159             // operational state, skip processing it
1160             InterfaceInfo ifInfo = interfaceManager
1161                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1162             if (!isOperational(ifInfo)) {
1163                 continue;
1164             }
1165
1166             if (!interfaceManager.isExternalInterface(ifName)) {
1167                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1168                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1169                 bucketId++;
1170             }
1171         }
1172
1173         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1174                 MDSALUtil.buildBucketLists(listBucket));
1175         LOG.trace("installing the localBroadCast Group:{}", group);
1176         mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group);
1177     }
1178
1179     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1180             InterfaceInfo interfaceInfo) {
1181         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1182         if (etreeInstance != null) {
1183             List<Bucket> listBucket = new ArrayList<>();
1184             int bucketId = 0;
1185
1186             List<String> interfaces = new ArrayList<>();
1187             if (newDpnInterface != null) {
1188                 interfaces = newDpnInterface.getInterfaces();
1189             }
1190             for (String ifName : interfaces) {
1191                 // In case if there is a InterfacePort in the cache which is not
1192                 // in
1193                 // operational state, skip processing it
1194                 InterfaceInfo ifInfo = interfaceManager
1195                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1196                 if (!isOperational(ifInfo)) {
1197                     continue;
1198                 }
1199
1200                 if (!interfaceManager.isExternalInterface(ifName)) {
1201                     // only add root interfaces
1202                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1203                 }
1204             }
1205
1206             if (listBucket.isEmpty()) { // No Buckets
1207                 createDropBucket(listBucket);
1208             }
1209
1210             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1211             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1212             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1213                     MDSALUtil.buildBucketLists(listBucket));
1214             LOG.trace("installing the localBroadCast Group:{}", group);
1215             mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group);
1216         }
1217     }
1218
1219     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1220             InterfaceInfo ifInfo) {
1221         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1222         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1223             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1224                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1225             bucketId++;
1226         }
1227         return bucketId;
1228     }
1229
1230     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1231             WriteTransaction deleteFlowGroupTx) {
1232         BigInteger dpnId = interfaceInfo.getDpId();
1233         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1234         List<Bucket> listBuckets = new ArrayList<>();
1235         int bucketId = 0;
1236         listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
1237         // listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, 1,
1238         // interfaceInfo));
1239         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1240                 MDSALUtil.buildBucketLists(listBuckets));
1241         LOG.trace("deleted the localBroadCast Group:{}", group);
1242         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1243     }
1244
1245     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1246             WriteTransaction deleteFlowGroupTx) {
1247         int bucketId = 0;
1248         int actionKey = 0;
1249         Long elanTag = elanInfo.getElanTag();
1250         List<Bucket> listBuckets = new ArrayList<>();
1251         List<Action> listAction = new ArrayList<>();
1252         listAction.add(new ActionGroup(++actionKey, ElanUtils.getElanLocalBCGId(elanTag)).buildAction());
1253         listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
1254                 MDSALUtil.WATCH_GROUP));
1255         bucketId++;
1256         listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, elanTag));
1257         BigInteger dpnId = interfaceInfo.getDpId();
1258         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1259         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1260                 MDSALUtil.buildBucketLists(listBuckets));
1261         LOG.trace("deleting the remoteBroadCast group:{}", group);
1262         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1263     }
1264
1265     /**
1266      * Installs a flow in the External Tunnel table consisting in translating
1267      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1268      * elanTag that will be used later in the ELANs pipeline.
1269      *
1270      * @param dpnId
1271      *            the dpn id
1272      * @param elanInfo
1273      *            the elan info
1274      */
1275     public void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1276         long elanTag = elanInfo.getElanTag();
1277         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1278                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1279                 elanInfo.getElanInstanceName(), // flowName
1280                 0, // idleTimeout
1281                 0, // hardTimeout
1282                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1283                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1284                 getInstructionsIntOrExtTunnelTable(elanTag));
1285
1286         mdsalManager.installFlow(flowEntity);
1287     }
1288
1289     /**
1290      * Removes, from External Tunnel table, the flow that translates from VNI to
1291      * elanTag. Important: ensure this method is only called whenever there is
1292      * no other ElanInterface in the specified DPN
1293      *
1294      * @param dpnId
1295      *            DPN whose Ext Tunnel table is going to be modified
1296      * @param elanInfo
1297      *            holds the elanTag needed for selecting the flow to be removed
1298      */
1299     public void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1300         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1301         // last ElanInstance plus
1302         // adding a new one does (almost at the same time) are executed in that
1303         // exact order
1304
1305         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1306         FlowEntity flowEntity = new FlowEntityBuilder()
1307             .setDpnId(dpnId)
1308             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1309             .setFlowId(flowId)
1310             .build();
1311         mdsalManager.removeFlow(flowEntity);
1312     }
1313
1314     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1315         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1316         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1317     }
1318
1319     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1320             WriteTransaction writeFlowGroupTx) {
1321         List<? extends MatchInfoBase> listMatchInfoBase;
1322         List<InstructionInfo> instructionInfos;
1323         long serviceId;
1324         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1325             serviceId = elanTag;
1326             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1327             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1328         } else {
1329             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1330             listMatchInfoBase = buildMatchesForVni(serviceId);
1331             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1332         }
1333         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1334                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1335                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1336                 instructionInfos);
1337         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1338     }
1339
1340     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1341             WriteTransaction writeFlowGroupTx) {
1342         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1343         if (etreeInstance != null) {
1344             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1345         }
1346     }
1347
1348     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1349         long elanTag = elanInfo.getElanTag();
1350         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1351         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1352         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1353     }
1354
1355     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1356             WriteTransaction writeFlowGroupTx) {
1357         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1358         if (etreeLeafTag != null) {
1359             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1360             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1361             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1362         }
1363     }
1364
1365     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1366             WriteTransaction writeFlowGroupTx) {
1367         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1368                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1369                 5, elanInfo.getElanInstanceName(), 0, 0,
1370                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1371                 getMatchesForElanTag(elanTag, /* SH flag */false),
1372                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1373
1374         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1375     }
1376
1377     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1378             WriteTransaction writeFlowGroupTx) {
1379         // only if ELAN can connect to external network, perform the following
1380
1381         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1382             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1383                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1384                     5, elanInfo.getElanInstanceName(), 0, 0,
1385                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1386                     getMatchesForElanTag(elanTag, /* SH flag */true),
1387                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1388             mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1389         }
1390     }
1391
1392
1393     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo, WriteTransaction deleteFlowGroupTx,
1394             long elanTag) {
1395         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1396                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1397         mdsalManager.removeFlowToTx(dpId, flow, deleteFlowGroupTx);
1398
1399         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1400             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1401                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1402                     .build();
1403             mdsalManager.removeFlowToTx(dpId, flow2, deleteFlowGroupTx);
1404         }
1405     }
1406
1407     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1408         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1409     }
1410
1411     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag, WriteTransaction tx) {
1412         if (isStandardElanService(elanInterface)) {
1413             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1414                     elanInterface.getName(), lportTag, tx);
1415         } else { // Etree service
1416             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1417         }
1418     }
1419
1420     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1421             WriteTransaction tx) {
1422         int instructionKey = 0;
1423         List<Instruction> instructions = new ArrayList<>();
1424         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1425                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1426
1427         List<Action> actions = new ArrayList<>();
1428         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1429                 lportTag).buildAction());
1430         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1431                 elanTag).buildAction());
1432         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1433
1434         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1435                 ++instructionKey));
1436
1437         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1438         BoundServices serviceInfo = ElanUtils.getBoundServices(
1439                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1440                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1441         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1442         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1443                 bindServiceId);
1444         if (!existingElanService.isPresent()) {
1445             tx.put(LogicalDatastoreType.CONFIGURATION, bindServiceId, serviceInfo,
1446                     WriteTransaction.CREATE_MISSING_PARENTS);
1447         }
1448     }
1449
1450     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1451             WriteTransaction tx) {
1452         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1453             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1454                     lportTag, tx);
1455         } else {
1456             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1457             if (etreeInstance == null) {
1458                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1459                         elanInterface.getName(), elanInfo.getElanInstanceName());
1460             } else {
1461                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1462                         elanInterface.getName(), lportTag, tx);
1463             }
1464         }
1465     }
1466
1467     private boolean isStandardElanService(ElanInterface elanInterface) {
1468         return elanInterface.augmentation(EtreeInterface.class) == null;
1469     }
1470
1471     protected void unbindService(String interfaceName, ReadWriteTransaction tx) throws ReadFailedException {
1472         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1473         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1474         if (tx.read(LogicalDatastoreType.CONFIGURATION, bindServiceId).checkedGet().isPresent()) {
1475             tx.delete(LogicalDatastoreType.CONFIGURATION, bindServiceId);
1476         }
1477     }
1478
1479     private String getFlowRef(long tableId, long elanTag) {
1480         return String.valueOf(tableId) + elanTag;
1481     }
1482
1483     private String getFlowRef(long tableId, long elanTag, String flowName) {
1484         return new StringBuffer().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1485                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1486     }
1487
1488     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1489         return String.valueOf(tableId) + elanTag + shFlag;
1490     }
1491
1492     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1493         List<Action> listAction = new ArrayList<>();
1494         int actionKey = 0;
1495         listAction.add(
1496             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1497         actionKey++;
1498         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1499         return listAction;
1500     }
1501
1502     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1503             List<String> interfaceNames, WriteTransaction tx) {
1504         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1505                 .withKey(new DpnInterfacesKey(dpId)).build();
1506         tx.put(LogicalDatastoreType.OPERATIONAL,
1507                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1508                 WriteTransaction.CREATE_MISSING_PARENTS);
1509         return dpnInterface;
1510     }
1511
1512     /**
1513      * Delete elan dpn interface from operational DS.
1514      *
1515      * @param elanInstanceName
1516      *            the elan instance name
1517      * @param dpId
1518      *            the dp id
1519      */
1520     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId, WriteTransaction tx) {
1521         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1522                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1523         Optional<DpnInterfaces> dpnInterfaces = ElanUtils.read(broker,
1524                 LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1525         if (dpnInterfaces.isPresent()) {
1526             tx.delete(LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1527         }
1528     }
1529
1530     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1531             WriteTransaction tx) {
1532         List<String> interfaceNames = new ArrayList<>();
1533         interfaceNames.add(interfaceName);
1534         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1535                 .withKey(new DpnInterfacesKey(dpId)).build();
1536         tx.put(LogicalDatastoreType.OPERATIONAL,
1537                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1538                 WriteTransaction.CREATE_MISSING_PARENTS);
1539         return dpnInterface;
1540     }
1541
1542     private void createElanInterfaceTablesList(String interfaceName, WriteTransaction tx) {
1543         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1544                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1545         Optional<ElanInterfaceMac> interfaceMacTables = ElanUtils.read(broker,
1546                 LogicalDatastoreType.OPERATIONAL, elanInterfaceMacTables);
1547         // Adding new Elan Interface Port to the operational DataStore without
1548         // Static-Mac Entries..
1549         if (!interfaceMacTables.isPresent()) {
1550             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1551                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1552             tx.put(LogicalDatastoreType.OPERATIONAL,
1553                     ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1554                     WriteTransaction.CREATE_MISSING_PARENTS);
1555         }
1556     }
1557
1558     private void createElanStateList(String elanInstanceName, String interfaceName, WriteTransaction tx) {
1559         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1560         Optional<Elan> elanInterfaceLists = ElanUtils.read(broker,
1561                 LogicalDatastoreType.OPERATIONAL, elanInstance);
1562         // Adding new Elan Interface Port to the operational DataStore without
1563         // Static-Mac Entries..
1564         if (elanInterfaceLists.isPresent()) {
1565             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1566             if (interfaceLists == null) {
1567                 interfaceLists = new ArrayList<>();
1568             }
1569             interfaceLists.add(interfaceName);
1570             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1571                     .withKey(new ElanKey(elanInstanceName)).build();
1572             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName),
1573                     elanState, WriteTransaction.CREATE_MISSING_PARENTS);
1574         }
1575     }
1576
1577     private boolean isOperational(InterfaceInfo interfaceInfo) {
1578         if (interfaceInfo == null) {
1579             return false;
1580         }
1581         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1582     }
1583
1584     @SuppressWarnings("checkstyle:IllegalCatch")
1585     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) {
1586         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1587         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1588                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1589         if (dpnInterfaceLists == null) {
1590             return;
1591         }
1592         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1593         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1594             int cnt = 0;
1595             String elanName = elanDpns.getElanInstanceName();
1596             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1597             if (elanInfo == null) {
1598                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1599                         + "skipping this ELAN for tunnel handling", elanName);
1600                 continue;
1601             }
1602             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1603                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1604                 continue;
1605             }
1606             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1607             if (dpnInterfaces == null) {
1608                 continue;
1609             }
1610             DpnInterfaces dstDpnIf = null;
1611             for (DpnInterfaces dpnIf : dpnInterfaces) {
1612                 BigInteger dpnIfDpId = dpnIf.getDpId();
1613                 if (dpnIfDpId.equals(srcDpId)) {
1614                     cnt++;
1615                 } else if (dpnIfDpId.equals(dstDpId)) {
1616                     cnt++;
1617                     dstDpnIf = dpnIf;
1618                 }
1619             }
1620             if (cnt == 2) {
1621                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1622                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1623                 jobCoordinator.enqueueJob(elanName, () -> {
1624                     // update Remote BC Group
1625                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1626                     try {
1627                         elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId);
1628                     } catch (RuntimeException e) {
1629                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1630                     }
1631                     Set<String> interfaceLists = new HashSet<>();
1632                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1633                     for (String ifName : interfaceLists) {
1634                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1635                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1636                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1637                             if (isOperational(interfaceInfo)) {
1638                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1639                             }
1640                             return Collections.emptyList();
1641                         }, ElanConstants.JOB_MAX_RETRIES);
1642                     }
1643                     return Collections.emptyList();
1644                 }, ElanConstants.JOB_MAX_RETRIES);
1645             }
1646
1647         }
1648     }
1649
1650     /**
1651      * Handle external tunnel state event.
1652      *
1653      * @param externalTunnel
1654      *            the external tunnel
1655      * @param intrf
1656      *            the interface
1657      */
1658     public void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1659         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1660             return;
1661         }
1662         // dpId/externalNodeId will be available either in source or destination
1663         // based on the tunnel end point
1664         BigInteger dpId = null;
1665         NodeId externalNodeId = null;
1666         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1667             dpId = new BigInteger(externalTunnel.getSourceDevice());
1668             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1669         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1670             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1671             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1672         }
1673         if (dpId == null || externalNodeId == null) {
1674             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1675             return;
1676         }
1677
1678         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1679         if (dpnInterfaceLists == null) {
1680             return;
1681         }
1682         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1683         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1684             String elanName = elanDpns.getElanInstanceName();
1685             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1686
1687             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1688             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1689                     || dpnInterfaces.getInterfaces().isEmpty()) {
1690                 continue;
1691             }
1692             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1693
1694             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpId);
1695             // install L2gwDevices local macs in dpn.
1696             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1697             // Install dpn macs on external device
1698             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1699                     externalNodeId);
1700         }
1701         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1702     }
1703
1704     /**
1705      * Installs dpn macs in external device. first it checks if the physical
1706      * locator towards this dpn tep is present or not if the physical locator is
1707      * present go ahead and add the ucast macs otherwise update the mcast mac
1708      * entry to include this dpn tep ip and schedule the job to put ucast macs
1709      * once the physical locator is programmed in device
1710      *
1711      * @param elanName
1712      *            the elan name
1713      * @param lstElanInterfaceNames
1714      *            the lst Elan interface names
1715      * @param dpnId
1716      *            the dpn id
1717      * @param externalNodeId
1718      *            the external node id
1719      */
1720     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1721             NodeId externalNodeId) {
1722         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1723                 externalNodeId.getValue());
1724         if (elanL2GwDevice == null) {
1725             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1726             return;
1727         }
1728         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1729         if (dpnTepIp == null) {
1730             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1731                     externalNodeId);
1732             return;
1733         }
1734
1735         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1736         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1737                 LogicalDatastoreType.OPERATIONAL);
1738         boolean phyLocAlreadyExists =
1739                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1740                 dpnTepIp);
1741         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1742                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1743         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1744
1745         if (phyLocAlreadyExists) {
1746             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1747             return;
1748         }
1749         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1750         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1751     }
1752
1753     /**
1754      * Validate external tunnel state event.
1755      *
1756      * @param externalTunnel
1757      *            the external tunnel
1758      * @param intrf
1759      *            the intrf
1760      * @return true, if successful
1761      */
1762     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1763         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1764             String srcDevice = externalTunnel.getDestinationDevice();
1765             String destDevice = externalTunnel.getSourceDevice();
1766             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1767                     LogicalDatastoreType.CONFIGURATION);
1768             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1769                     otherEndPointExtTunnel);
1770             if (otherEndPointExtTunnel != null) {
1771                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1772                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1773                 if (otherEndPointInterfaceOperational) {
1774                     return true;
1775                 } else {
1776                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1777                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1778                 }
1779             }
1780         }
1781         return false;
1782     }
1783
1784     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1785         List<MatchInfo> mkMatches = new ArrayList<>();
1786         // Matching metadata
1787         mkMatches.add(
1788                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1789         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1790         return mkMatches;
1791     }
1792
1793     @Override
1794     protected ElanInterfaceManager getDataTreeChangeListener() {
1795         return this;
1796     }
1797
1798     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1799                                              BigInteger dpId) {
1800         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1801         elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId);
1802         try {
1803             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1804         } catch (InterruptedException e) {
1805             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1806         }
1807     }
1808 }