elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.infrautils.utils.concurrent.LoggingFutures.addErrorLogging;
  15 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  16
  17 import com.google.common.base.Optional;
  18 import com.google.common.base.Preconditions;
  19 import com.google.common.collect.Lists;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import java.math.BigInteger;
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Queue;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.apache.commons.lang3.StringUtils;
  38 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.Datastore.Operational;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TransactionAdapter;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  50 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  51 import org.opendaylight.genius.itm.globals.ITMConstants;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  67 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  68 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.JvmGlobalLocks;
  72 import org.opendaylight.genius.utils.ServiceIndex;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  75 import org.opendaylight.infrautils.utils.concurrent.NamedSimpleReentrantLock.Acquired;
  76 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  77 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  78 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  79 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  80 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  81 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  82 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  83 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  84 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  85 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  86 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  87 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  88 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  89 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  90 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  91 import org.opendaylight.serviceutils.srm.RecoverableListener;
  92 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  94 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  95 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.GroupKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 142 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 143 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 144 import org.opendaylight.yangtools.yang.common.Uint32;
 145 import org.opendaylight.yangtools.yang.common.Uint64;
 146 import org.slf4j.Logger;
 147 import org.slf4j.LoggerFactory;
 148
 149 /**
 150  * Class in charge of handling creations, modifications and removals of
 151  * ElanInterfaces.
 152  *
 153  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 154  */
 155 @Singleton
 156 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 157         implements RecoverableListener {
 158     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 159     public static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 160     private static final boolean SH_FLAG_SET = true;
 161     private static final boolean SH_FLAG_UNSET = false;
 162
 163     private final DataBroker broker;
 164     private final ManagedNewTransactionRunner txRunner;
 165     private final IMdsalApiManager mdsalManager;
 166     private final IInterfaceManager interfaceManager;
 167     private final IdManagerService idManager;
 168     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 169     private final INeutronVpnManager neutronVpnManager;
 170     private final ElanItmUtils elanItmUtils;
 171     private final ElanEtreeUtils elanEtreeUtils;
 172     private final ElanL2GatewayUtils elanL2GatewayUtils;
 173     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 174     private final ElanUtils elanUtils;
 175     private final JobCoordinator jobCoordinator;
 176     private final ElanInstanceCache elanInstanceCache;
 177     private final ElanInterfaceCache elanInterfaceCache;
 178     private final ElanGroupCache elanGroupCache;
 179
 180     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 181         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 182
 183     @Inject
 184     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 185                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 186                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 187                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 188                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 189                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 190                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 191                                 final ElanInstanceCache elanInstanceCache,
 192                                 final ElanInterfaceCache elanInterfaceCache,
 193                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 194                                 ElanGroupCache elanGroupCache,
 195                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 196         super(ElanInterface.class, ElanInterfaceManager.class);
 197         this.broker = dataBroker;
 198         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 199         this.idManager = managerService;
 200         this.mdsalManager = mdsalApiManager;
 201         this.interfaceManager = interfaceManager;
 202         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 203         this.neutronVpnManager = neutronVpnManager;
 204         this.elanItmUtils = elanItmUtils;
 205         this.elanEtreeUtils = elanEtreeUtils;
 206         this.elanL2GatewayUtils = elanL2GatewayUtils;
 207         this.elanUtils = elanUtils;
 208         this.jobCoordinator = jobCoordinator;
 209         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 210         this.elanInstanceCache = elanInstanceCache;
 211         this.elanInterfaceCache = elanInterfaceCache;
 212         this.elanGroupCache = elanGroupCache;
 213         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 214     }
 215
 216     @Override
 217     @PostConstruct
 218     public void init() {
 219         registerListener();
 220     }
 221
 222     @Override
 223     public void registerListener() {
 224         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 225     }
 226
 227     @Override
 228     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 229         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 230     }
 231
 232     @Override
 233     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 234         String interfaceName = del.getName();
 235         String elanInstanceName = del.getElanInstanceName();
 236         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 237         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 238             elanInterfaces.remove(del);
 239             if (elanInterfaces.isEmpty()) {
 240                 unProcessedElanInterfaces.remove(elanInstanceName);
 241             }
 242         }
 243         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 244         /*
 245          * Handling in case the elan instance is deleted.If the Elan instance is
 246          * deleted, there is no need to explicitly delete the elan interfaces
 247          */
 248         if (elanInfo == null) {
 249             return;
 250         }
 251         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 252         if (interfaceInfo == null && elanInfo.isExternal()) {
 253             // In deleting external network, the underlying ietf Inteface might have been removed
 254             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 255             // from Operational DS instead
 256             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 257         }
 258         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 259                 interfaceName, interfaceInfo, this);
 260         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 261     }
 262
 263     private static class RemoveElanInterfaceHolder {
 264         boolean isLastElanInterface = false;
 265         Uint64 dpId = null;
 266     }
 267
 268     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 269     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 270             InterfaceInfo interfaceInfo) {
 271         String elanName = elanInfo.getElanInstanceName();
 272         Uint32 elanTag = elanInfo.getElanTag();
 273         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 274         List<ListenableFuture<Void>> futures = new ArrayList<>();
 275         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 276         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 277             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 278             if (elanState == null) {
 279                 return;
 280             }
 281             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 282                 List<String> elanInterfaces = elanState.getElanInterfaces();
 283                 if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 284                     holder.isLastElanInterface = true;
 285                 }
 286                 if (interfaceInfo != null) {
 287                     holder.dpId = interfaceInfo.getDpId();
 288                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 289                         interfaceName, elanTag, interfaceTx);
 290                     /*
 291                      * If there are not elan ports, remove the unknown dmac, terminating
 292                      * service table flows, remote/local bc group
 293                      */
 294                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 295                         || dpnInterfaces.getInterfaces().isEmpty()) {
 296                         // No more Elan Interfaces in this DPN
 297                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 298                             holder.dpId);
 299                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 300                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag().toJava());
 301                         }
 302                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag().toJava());
 303                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 304                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 305                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 306                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 307                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 308                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 309                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 310                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 311                             }
 312                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 313                         }
 314                     } else {
 315                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 316                     }
 317                 }
 318             }));
 319         }));
 320         futures.forEach(ElanUtils::waitForTransactionToComplete);
 321
 322         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 323                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 324         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 325                 ElanConstants.JOB_MAX_RETRIES);
 326
 327         return futures;
 328     }
 329
 330     private void removeEtreeUnknownDmacFlow(Uint64 dpId, ElanInstance elanInfo,
 331             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 332             throws ExecutionException, InterruptedException {
 333         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag().toJava());
 334         if (etreeLeafTag != null) {
 335             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue().toJava();
 336             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 337         }
 338     }
 339
 340     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 341             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 342             throws ExecutionException, InterruptedException {
 343         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 344         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 345     }
 346
 347     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 348             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 349             throws ExecutionException, InterruptedException {
 350         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 351         if (etreeInstance != null) {
 352             Uint64 dpnId = interfaceInfo.getDpId();
 353             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue().toJava());
 354             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 355             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 356         }
 357     }
 358
 359     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 360             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 361             throws ExecutionException, InterruptedException {
 362         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 363         if (etreeInstance != null) {
 364             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue().toJava();
 365             Uint64 dpnId = interfaceInfo.getDpId();
 366             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 367             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 368             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 369         }
 370     }
 371
 372     private static Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 373             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 374         String elanName = elanInfo.getElanInstanceName();
 375         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 376         if (elanState == null) {
 377             return elanState;
 378         }
 379         List<String> existingElanInterfaces = elanState.getElanInterfaces();
 380         List<String> elanInterfaces = new ArrayList<>();
 381         existingElanInterfaces.forEach(iface -> elanInterfaces.add(iface));
 382         boolean isRemoved = elanInterfaces != null && elanInterfaces.remove(interfaceName);
 383         if (!isRemoved) {
 384             return elanState;
 385         }
 386
 387         if (elanInterfaces.isEmpty()) {
 388             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 389             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 390             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 391             tx.delete(ElanUtils.getElanDpnOperationDataPath(elanName));
 392         } else {
 393             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 394                     .withKey(new ElanKey(elanName)).build();
 395             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 396         }
 397         return elanState;
 398     }
 399
 400     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 401             throws ReadFailedException {
 402         // removing the ElanInterface from the config data_store if interface is
 403         // not present in Interface config DS
 404         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 405             interfaceName) == null
 406                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 407             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 408         }
 409     }
 410
 411     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 412             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 413         String elanName = elanInfo.getElanInstanceName();
 414         List<ListenableFuture<Void>> futures = new ArrayList<>();
 415         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 416             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 417                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 418                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 419                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 420                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 421                 if (interfaceInfo != null) {
 422                     if (existingElanInterfaceMac.isPresent()) {
 423                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 424                         if (existingMacEntries != null) {
 425                             List<PhysAddress> macAddresses = new ArrayList<>();
 426                             for (MacEntry macEntry : existingMacEntries) {
 427                                 PhysAddress macAddress = macEntry.getMacAddress();
 428                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 429                                         macAddress.getValue(), interfaceName);
 430                                 Optional<MacEntry> macEntryOptional =
 431                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 432                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 433                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 434                                 }
 435                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 436                                 macAddresses.add(macAddress);
 437                             }
 438
 439                             // Removing all those MACs from External Devices belonging
 440                             // to this ELAN
 441                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 442                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 443                             }
 444                         }
 445                     }
 446                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 447                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 448                 } else if (existingElanInterfaceMac.isPresent()) {
 449                     // Interface does not exist in ConfigDS, so lets remove everything
 450                     // about that interface related to Elan
 451                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 452                     if (macEntries != null) {
 453                         for (MacEntry macEntry : macEntries) {
 454                             PhysAddress macAddress = macEntry.getMacAddress();
 455                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 456                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 457                             }
 458                         }
 459                     }
 460                 }
 461                 if (existingElanInterfaceMac.isPresent()) {
 462                     interfaceTx.delete(elanInterfaceId);
 463                 }
 464                 unbindService(interfaceName, flowTx);
 465                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 466             }));
 467         }));
 468         return futures;
 469     }
 470
 471     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, Uint64 dpId,
 472                                                                          String interfaceName, Uint32 elanTag,
 473                                                                          TypedReadWriteTransaction<Operational> tx)
 474             throws ExecutionException, InterruptedException {
 475         // FIXME: pass in and use ElanInstanceKey instead?
 476         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 477         lock.lock();
 478         try {
 479             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 480             if (dpnInterfaces != null) {
 481                 List<String> interfaceLists = dpnInterfaces.getInterfaces();
 482                 if (interfaceLists != null) {
 483                     interfaceLists.remove(interfaceName);
 484                 }
 485
 486                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 487                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 488                     deleteElanDpnInterface(elanName, dpId, tx);
 489                 } else {
 490                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 491                 }
 492             }
 493             return dpnInterfaces;
 494         } finally {
 495             lock.unlock();
 496         }
 497     }
 498
 499     private void deleteAllRemoteMacsInADpn(String elanName, Uint64 dpId, Uint32 elanTag) {
 500         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 501         addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 502             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 503                 Uint64 currentDpId = dpnInterface.getDpId();
 504                 if (!currentDpId.equals(dpId) && dpnInterface.getInterfaces() != null) {
 505                     for (String elanInterface : dpnInterface.getInterfaces()) {
 506                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 507                         if (macs == null || macs.getMacEntry() == null) {
 508                             continue;
 509                         }
 510                         for (MacEntry mac : macs.getMacEntry()) {
 511                             removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 512                             removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 513                         }
 514                     }
 515                 }
 516             }
 517         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 518     }
 519
 520     private void removeEtreeMacFlowInTheDPN(Uint64 dpId, Uint32 elanTag, Uint64 currentDpId, MacEntry mac,
 521             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 522         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag.longValue());
 523         if (etreeLeafTag != null) {
 524             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(),
 525                 currentDpId, mac, confTx);
 526         }
 527     }
 528
 529     private void removeTheMacFlowInTheDPN(Uint64 dpId, Uint32 elanTag, Uint64 currentDpId, MacEntry mac,
 530             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 531         mdsalManager
 532                 .removeFlow(confTx, dpId,
 533                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 534                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 535                                         mac.getMacAddress().getValue(), elanTag)));
 536     }
 537
 538     /*
 539     * Possible Scenarios for update
 540     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 541         then all {1,2,3,4} should be removed
 542
 543         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 544         then all {1,2,3,4} should be added
 545
 546         c. if orig = {1,2,3,4} updated={2,3,4}
 547         then 1 should be removed
 548
 549         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 550         then we should just add 5
 551
 552         e. if orig = {1,2,3,4} updated={2,3,4,5}
 553         then 1 should be removed , 5 should be added
 554     * */
 555     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 556     @Override
 557     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 558         // updating the static-Mac Entries for the existing elanInterface
 559         String elanName = update.getElanInstanceName();
 560         String interfaceName = update.getName();
 561         LOG.info("Update static mac entries for elan interface {} in elan instance {}", interfaceName, elanName);
 562
 563         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 564         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 565         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 566         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 567
 568         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 569                 deletedEntry.getMacAddress()));
 570
 571         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 572         * Hence add the macentries for the same.*/
 573         for (StaticMacEntries staticMacEntry : updatedEntries) {
 574             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 575                     staticMacEntry.getMacAddress());
 576             addErrorLogging(ElanUtils.waitForTransactionToComplete(
 577                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 578                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 579                     if (existingMacEntry.isPresent()) {
 580                         LOG.debug("updating elan interface forwarding table for mac entry {} elan instance {}",
 581                                 existingMacEntry.get(), elanName);
 582                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 583                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 584                             tx);
 585                     } else {
 586                         LOG.info("Adding elan interface forwarding table for mac entry {} elan interface"
 587                                 + " {} elan instance {}.", staticMacEntry.getMacAddress(), interfaceName, elanName);
 588                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 589                             elanName, interfaceName, staticMacEntry, tx);
 590                     }
 591                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 592         }
 593     }
 594
 595     @Override
 596     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 597         LOG.info("Init for ELAN interface Add {}", elanInterfaceAdded);
 598         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 599             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 600             String interfaceName = elanInterfaceAdded.getName();
 601             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 602             if (interfaceInfo == null) {
 603                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 604                 return;
 605             }
 606             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 607
 608             if (elanInstance == null) {
 609                 // Add the ElanInstance in the Configuration data-store
 610                 List<String> elanInterfaces = new ArrayList<>();
 611                 elanInterfaces.add(interfaceName);
 612                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 613                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 614                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 615                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 616             }
 617
 618             Uint32 elanTag = elanInstance.getElanTag();
 619             // If elan tag is not updated, then put the elan interface into
 620             // unprocessed entry map and entry. Let entries
 621             // in this map get processed during ELAN update DCN.
 622             if (elanTag == null || elanTag.longValue() == 0L) {
 623                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 624                 if (elanInterfaces == null) {
 625                     elanInterfaces = new ConcurrentLinkedQueue<>();
 626                 }
 627                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 628                     elanInterfaces.add(elanInterfaceAdded);
 629                 }
 630                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 631                 return;
 632             }
 633             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 634                     interfaceInfo, elanInstance, this);
 635             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 636         }), LOG, "Error processing added ELAN interface");
 637     }
 638
 639     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 640         LOG.trace("Handling unprocessed elan interfaces for elan instance {}", elanInstance.getElanInstanceName());
 641         List<ListenableFuture<Void>> futures = new ArrayList<>();
 642         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 643         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 644             return futures;
 645         }
 646         for (ElanInterface elanInterface : elanInterfaces) {
 647             String interfaceName = elanInterface.getName();
 648             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 649             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 650         }
 651         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 652         LOG.info("Unprocessed elan interfaces for elan instance {} has been removed.",
 653                 elanInstance.getElanInstanceName());
 654         return futures;
 655     }
 656
 657     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 658             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 659         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 660                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 661         List<DpnInterfaces> dpnInterfaceLists = null;
 662         if (elanDpnInterfacesList != null) {
 663             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 664         }
 665         if (dpnInterfaceLists == null) {
 666             dpnInterfaceLists = new ArrayList<>();
 667         }
 668         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 669             Uint64 dstDpId = interfaceInfo.getDpId();
 670             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 671                 continue;
 672             }
 673             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 674             for (String remoteIf : remoteElanInterfaces) {
 675                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 676                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 677                 if (elanIfMac == null || remoteInterface == null) {
 678                     continue;
 679                 }
 680                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 681                 if (remoteMacEntries != null) {
 682                     for (MacEntry macEntry : remoteMacEntries) {
 683                         String macAddress = macEntry.getMacAddress().getValue();
 684                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 685                                 dstDpId, elanInstance.getElanInstanceName());
 686                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 687                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 688                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 689                     }
 690                 }
 691             }
 692         }
 693     }
 694
 695     private static class AddElanInterfaceHolder {
 696         private DpnInterfaces dpnInterfaces = null;
 697         private boolean isFirstInterfaceInDpn = false;
 698         private Uint64 dpId;
 699     }
 700
 701     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 702     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 703             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 704         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 705         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 706         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 707
 708         String interfaceName = elanInterface.getName();
 709         String elanInstanceName = elanInterface.getElanInstanceName();
 710         LOG.trace("Adding elan interface: interface name {} , instance name {}", interfaceName, elanInstanceName);
 711
 712         List<ListenableFuture<Void>> futures = new ArrayList<>();
 713         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 714         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 715             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 716             if (elanInfo == null) {
 717                 LOG.trace("elanInfo is null for elan instance: {}", elanInstanceName);
 718                 List<String> elanInterfaces = new ArrayList<>();
 719                 elanInterfaces.add(interfaceName);
 720                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 721                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 722                         operTx)));
 723             } else {
 724                 createElanStateList(elanInstanceName, interfaceName, operTx);
 725             }
 726             // Specific actions to the DPN where the ElanInterface has been added,
 727             // for example, programming the
 728             // External tunnel table if needed or adding the ElanInterface to the
 729             // DpnInterfaces in the operational DS.
 730             holder.dpId = interfaceInfo.getDpId();
 731             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 732                 // FIXME: use elanInstaince.key() instead?
 733                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 734                 lock.lock();
 735                 try {
 736                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 737                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 738                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 739                     if (ElanUtils.isVlan(elanInstance)) {
 740                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 741                             elanInstanceName, existingElanDpnInterfaces);
 742                     } else {
 743                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 744                     }
 745                     if (holder.isFirstInterfaceInDpn) {
 746                         // ELAN's 1st ElanInterface added to this DPN
 747                         if (!existingElanDpnInterfaces.isPresent()) {
 748                             holder.dpnInterfaces =
 749                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 750                         } else {
 751                             List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 752                             List<String> elanInterfaces =
 753                                 existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 754                             elanInterfaces.add(interfaceName);
 755                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 756                                 elanInterfaces, operTx);
 757                         }
 758                         LOG.debug("1st interface {} for elan {} is added to dpn {}",
 759                                 interfaceName, elanInstanceName, holder.dpId);
 760                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 761                         // table, but only if Elan has VNI
 762                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 763                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 764                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 765                         }
 766                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 767                             interfaceName);
 768                     } else {
 769                         List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 770                         List<String> elanInterfaces =
 771                             existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 772                         elanInterfaces.add(interfaceName);
 773                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 774                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 775                                 interfaceName);
 776                         }
 777                         holder.dpnInterfaces =
 778                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 779                         LOG.debug("Interface {} for elan {} is added to dpn {}",
 780                                 interfaceName, elanInstanceName, holder.dpId);
 781                     }
 782                 } finally {
 783                     lock.unlock();
 784                 }
 785             }
 786
 787             // add code to install Local/Remote BC group, unknow DMAC entry,
 788             // terminating service table flow entry
 789             // call bindservice of interfacemanager to create ingress table flow
 790             // enty.
 791             // Add interface to the ElanInterfaceForwardingEntires Container
 792             createElanInterfaceTablesList(interfaceName, operTx);
 793         }));
 794         futures.forEach(ElanUtils::waitForTransactionToComplete);
 795         futures.add(
 796             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 797                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 798                     holder.isFirstInterfaceInDpn, confTx))));
 799
 800         // add the vlan provider interface to remote BC group for the elan
 801         // for internal vlan networks
 802         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 803             if (interfaceManager.isExternalInterface(interfaceName)) {
 804                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 805                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 806             }
 807         }
 808         if (holder.isFirstInterfaceInDpn) {
 809             // ELAN's 1st ElanInterface added to this DPN
 810             LOG.debug("Adding dpn into operational dpn list {}", holder.dpId);
 811             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 812                 operTx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId),
 813                         holder.dpnInterfaces, CREATE_MISSING_PARENTS);
 814             }));
 815         } else {
 816             LOG.debug("Updated dpn into operational dpn list {}", holder.dpId);
 817         }
 818
 819         scheduleElanInterfaceWorkerAfterRemoteBcGroup(elanInstance, interfaceInfo, holder.dpnInterfaces,
 820                 holder.isFirstInterfaceInDpn, elanInterface);
 821         return futures;
 822     }
 823
 824     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 825     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 826             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 827         String elanInstanceName = elanInstance.getElanInstanceName();
 828         String interfaceName = elanInterface.getName();
 829         List<ListenableFuture<Void>> futures = new ArrayList<>();
 830         Uint64 dpId = interfaceInfo.getDpId();
 831         boolean isInterfaceOperational = isOperational(interfaceInfo);
 832         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 833             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 834                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 835                     isFirstInterfaceInDpn, confTx);
 836
 837                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 838                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 839
 840                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 841                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 842                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 843                             staticMacEntry.getMacAddress());
 844                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 845                             LogicalDatastoreType.OPERATIONAL, macId);
 846                         if (existingMacEntry.isPresent()) {
 847                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 848                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 849                                 existingMacEntry.get(), operTx);
 850                         } else {
 851                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 852                                 interfaceName, staticMacEntry, operTx);
 853                         }
 854
 855                         if (isInterfaceOperational) {
 856                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 857                             // DPNs
 858                             String macAddress = staticMacEntry.getMacAddress().getValue();
 859                             LOG.info(
 860                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 861                                     + " {}",
 862                                 macAddress, elanInstanceName, interfaceName);
 863                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 864                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 865                         }
 866                     }
 867
 868                     if (isInterfaceOperational) {
 869                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 870                         // on purpose.
 871                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 872                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 873                         }
 874                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 875                             staticMacAddresses);
 876                     }
 877                 }
 878             }));
 879         }));
 880         futures.forEach(ElanUtils::waitForTransactionToComplete);
 881         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 882             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 883             try {
 884                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 885                 if (neutronPort != null) {
 886                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 887                 }
 888             } catch (IllegalArgumentException ex) {
 889                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 890             }
 891         }
 892         return futures;
 893     }
 894
 895     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 896             PhysAddress physAddress) {
 897         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 898         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 899         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 900                 LogicalDatastoreType.OPERATIONAL, macId);
 901
 902         if (!existingMacEntry.isPresent()) {
 903             return;
 904         }
 905
 906         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 907                 .withKey(new MacEntryKey(physAddress)).build();
 908         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 909                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 910     }
 911
 912     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 913             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 914         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 915         if (!existingElanDpnInterfaces.isPresent()) {
 916             return true;
 917         }
 918         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 919             return false;
 920         }
 921         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 922         int dummyInterfaceCount =  0;
 923         List<String> interfaces = dpnInterfaces.getInterfaces();
 924         if (interfaces == null) {
 925             return true;
 926         }
 927         if (interfaces.contains(routerPortUuid)) {
 928             dummyInterfaceCount++;
 929         }
 930         if (interfaces.contains(elanInstanceName)) {
 931             dummyInterfaceCount++;
 932         }
 933         return interfaces.size() == dummyInterfaceCount;
 934     }
 935
 936     private static InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName,
 937             PhysAddress physAddress) {
 938         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 939                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 940     }
 941
 942     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 943             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 944         if (!isOperational(interfaceInfo)) {
 945             LOG.warn("Interface {} is not operational", elanInterface.getName());
 946             return;
 947         }
 948         Uint64 dpId = interfaceInfo.getDpId();
 949         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 950             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 951         }
 952         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 953         if (isFirstInterfaceInDpn) {
 954             // Terminating Service , UnknownDMAC Table.
 955             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 956             // for ELAN Instance is VxLAN
 957             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 958                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 959             }
 960             setupUnknownDMacTable(elanInstance, dpId, confTx);
 961             /*
 962              * Install remote DMAC flow. This is required since this DPN is
 963              * added later to the elan instance and remote DMACs of other
 964              * interfaces in this elan instance are not present in the current
 965              * dpn.
 966              */
 967             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 968                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 969                         elanInstance.getElanInstanceName());
 970                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 971             }
 972         }
 973         // bind the Elan service to the Interface
 974         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 975     }
 976
 977     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 978             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 979         if (!isOperational(interfaceInfo)) {
 980             LOG.warn("Entries for interface on dpn {} not installed since interface {} is not operational",
 981                     dpnInterfaces.getDpId(), interfaceInfo.getInterfaceName());
 982             return;
 983         }
 984         // LocalBroadcast Group creation with elan-Interfaces
 985         LOG.info("Installing entries for interface {} on dpn {}", interfaceInfo.getInterfaceName(),
 986                 dpnInterfaces.getDpId());
 987         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
 988         if (isFirstInterfaceInDpn) {
 989             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 990             try {
 991                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 992             } catch (InterruptedException e1) {
 993                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 994             }
 995         }
 996     }
 997
 998     public InstanceIdentifier<Group> getGroupIid(ElanInstance elanInfo, Uint64 dpnId) {
 999         long remoteBcGroupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag().toJava());
1000         return InstanceIdentifier.builder(Nodes.class)
1001                 .child(Node.class, new NodeKey(new org.opendaylight.yang.gen.v1.urn.opendaylight
1002                         .inventory.rev130819.NodeId("openflow:" + dpnId.toString())))
1003                 .augmentation(FlowCapableNode.class)
1004                 .child(Group.class, new GroupKey(new GroupId(remoteBcGroupId))).build();
1005     }
1006
1007     public void scheduleElanInterfaceWorkerAfterRemoteBcGroup(ElanInstance elanInfo,
1008                                                               InterfaceInfo interfaceInfo,
1009                                                               DpnInterfaces dpnInterfaces,
1010                                                               boolean isFirstInterfaceInDpn,
1011                                                               ElanInterface elanInterface) {
1012         if (!isOperational(interfaceInfo)) {
1013             LOG.debug("Interface {} is not operational", elanInterface.getName());
1014             return;
1015         }
1016         String elanInterfaceJobKey = ElanUtils.getElanInterfaceJobKey(interfaceInfo.getInterfaceName());
1017         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(elanInterfaceJobKey,
1018                 elanInterface, interfaceInfo, elanInfo, isFirstInterfaceInDpn, this);
1019         InstanceIdentifier<Group> groupInstanceId = getGroupIid(elanInfo, dpnInterfaces.getDpId());
1020         elanGroupCache.addJobToWaitList(groupInstanceId, () -> {
1021             jobCoordinator.enqueueJob(elanInterfaceJobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
1022         });
1023     }
1024
1025     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1026             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1027         int ifTag = interfaceInfo.getInterfaceTag();
1028         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1029                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
1030                 0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1031                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
1032                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
1033
1034         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
1035         LOG.trace("Filter equals table(55) flow entry created on dpn: {} for interface port: {}",
1036                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1037
1038         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1039                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
1040                 0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1041                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1042
1043         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
1044         LOG.trace("Filter equals table(55) drop flow entry created on dpn: {} for interface port: {}",
1045                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1046     }
1047
1048     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1049             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
1050         int ifTag = interfaceInfo.getInterfaceTag();
1051         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1052             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
1053
1054         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1055
1056         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1057             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
1058             0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1059             getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1060
1061         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1062     }
1063
1064     private static List<MatchInfo> buildMatchesForVni(Uint64 vni) {
1065         List<MatchInfo> mkMatches = new ArrayList<>();
1066         MatchInfo match = new MatchTunnelId(vni);
1067         mkMatches.add(match);
1068         return mkMatches;
1069     }
1070
1071     private static List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1072         List<InstructionInfo> mkInstructions = new ArrayList<>();
1073         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1074         return mkInstructions;
1075     }
1076
1077     private static List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1078         List<MatchInfo> mkMatches = new ArrayList<>();
1079         // Matching metadata
1080         mkMatches.add(new MatchMetadata(
1081                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1082         return mkMatches;
1083     }
1084
1085     /**
1086      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1087      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1088      *
1089      * @param elanTag
1090      *            elanTag to be written in metadata when flow is selected
1091      * @return the instructions ready to be installed in a flow
1092      */
1093     private static List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Uint32 elanTag) {
1094         List<InstructionInfo> mkInstructions = new ArrayList<>();
1095         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag.longValue()),
1096                 ElanHelper.getElanMetadataMask()));
1097         /* applicable for EXTERNAL_TUNNEL_TABLE only
1098         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1099         */
1100         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1101         return mkInstructions;
1102     }
1103
1104     // Install DMAC entry on dst DPN
1105     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1106             Uint64 dstDpId) {
1107         String interfaceName = interfaceInfo.getInterfaceName();
1108         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1109         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1110             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1111             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1112                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1113                     for (MacEntry macEntry : macEntries) {
1114                         String macAddress = macEntry.getMacAddress().getValue();
1115                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1116                             interfaceName);
1117                         try (Acquired lock = ElanUtils.lockElanMacDPN(elanInfo.getElanTag().toJava(), macAddress,
1118                             interfaceInfo.getDpId())) {
1119                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1120                                 macAddress);
1121                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1122                         }
1123                     }
1124                 })));
1125         }
1126         return emptyList();
1127     }
1128
1129     private static void createDropBucket(List<Bucket> listBucket) {
1130         List<Action> actionsInfos = new ArrayList<>();
1131         actionsInfos.add(new ActionDrop().buildAction());
1132         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1133                 MDSALUtil.WATCH_GROUP);
1134         listBucket.add(dropBucket);
1135     }
1136
1137     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1138             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1139         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1140         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1141     }
1142
1143     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1144             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1145         List<Bucket> listBucket = new ArrayList<>();
1146         int bucketId = 0;
1147         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag().toJava());
1148
1149         List<String> interfaces = new ArrayList<>();
1150         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1151             interfaces = newDpnInterface.getInterfaces();
1152         }
1153         for (String ifName : interfaces) {
1154             // In case if there is a InterfacePort in the cache which is not in
1155             // operational state, skip processing it
1156             InterfaceInfo ifInfo = interfaceManager
1157                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1158             if (!isOperational(ifInfo)) {
1159                 continue;
1160             }
1161
1162             if (!interfaceManager.isExternalInterface(ifName)) {
1163                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1164                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1165                 bucketId++;
1166             }
1167         }
1168
1169         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1170                 MDSALUtil.buildBucketLists(listBucket));
1171         LOG.trace("installing the localBroadCast Group:{}", group);
1172         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1173     }
1174
1175     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1176             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1177         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1178         if (etreeInstance != null) {
1179             List<Bucket> listBucket = new ArrayList<>();
1180             int bucketId = 0;
1181
1182             List<String> interfaces = new ArrayList<>();
1183             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1184                 interfaces = newDpnInterface.getInterfaces();
1185             }
1186             for (String ifName : interfaces) {
1187                 // In case if there is a InterfacePort in the cache which is not
1188                 // in
1189                 // operational state, skip processing it
1190                 InterfaceInfo ifInfo = interfaceManager
1191                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1192                 if (!isOperational(ifInfo)) {
1193                     continue;
1194                 }
1195
1196                 if (!interfaceManager.isExternalInterface(ifName)) {
1197                     // only add root interfaces
1198                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1199                 }
1200             }
1201
1202             if (listBucket.isEmpty()) { // No Buckets
1203                 createDropBucket(listBucket);
1204             }
1205
1206             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().toJava();
1207             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1208             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1209                     MDSALUtil.buildBucketLists(listBucket));
1210             LOG.trace("installing the localBroadCast Group:{}", group);
1211             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1212         }
1213     }
1214
1215     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1216             InterfaceInfo ifInfo) {
1217         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1218         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1219             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1220                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1221             bucketId++;
1222         }
1223         return bucketId;
1224     }
1225
1226     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1227             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1228             throws ExecutionException, InterruptedException {
1229         Uint64 dpnId = interfaceInfo.getDpId();
1230         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag().toJava());
1231         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1232         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1233     }
1234
1235     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1236             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1237             throws ExecutionException, InterruptedException {
1238         Uint64 dpnId = interfaceInfo.getDpId();
1239         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag().toJava());
1240         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1241         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1242     }
1243
1244     /**
1245      * Installs a flow in the External Tunnel table consisting in translating
1246      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1247      * elanTag that will be used later in the ELANs pipeline.
1248      * @param dpnId the dpn id
1249      */
1250     private void setExternalTunnelTable(Uint64 dpnId, ElanInstance elanInfo,
1251             TypedWriteTransaction<Configuration> confTx) {
1252         Uint32 elanTag = elanInfo.getElanTag();
1253         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1254                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag.longValue()), 5, // prio
1255                 elanInfo.getElanInstanceName(), // flowName
1256                 0, // idleTimeout
1257                 0, // hardTimeout
1258                 Uint64.valueOf(ITMConstants.COOKIE_ITM_EXTERNAL.toJava().add(BigInteger.valueOf(elanTag.longValue()))),
1259                 buildMatchesForVni(Uint64.valueOf(ElanUtils.getVxlanSegmentationId(elanInfo).longValue())),
1260                 getInstructionsIntOrExtTunnelTable(elanTag));
1261
1262         mdsalManager.addFlow(confTx, flowEntity);
1263     }
1264
1265     /**
1266      * Removes, from External Tunnel table, the flow that translates from VNI to
1267      * elanTag. Important: ensure this method is only called whenever there is
1268      * no other ElanInterface in the specified DPN
1269      *
1270      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1271      */
1272     private void unsetExternalTunnelTable(Uint64 dpnId, ElanInstance elanInfo,
1273             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1274         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1275         // last ElanInstance plus
1276         // adding a new one does (almost at the same time) are executed in that
1277         // exact order
1278
1279         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag().toJava());
1280         FlowEntity flowEntity = new FlowEntityBuilder()
1281             .setDpnId(dpnId)
1282             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1283             .setFlowId(flowId)
1284             .build();
1285         mdsalManager.removeFlow(confTx, flowEntity);
1286     }
1287
1288     public void setupTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId,
1289             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1290         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1291         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1292     }
1293
1294     public void setupTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId, Uint32 elanTag,
1295             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1296         List<? extends MatchInfoBase> listMatchInfoBase;
1297         List<InstructionInfo> instructionInfos;
1298         Uint32 serviceId;
1299         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1300             serviceId = elanTag;
1301             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId(elanTag);
1302             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag.longValue()));
1303         } else {
1304             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1305             listMatchInfoBase = buildMatchesForVni(Uint64.valueOf(serviceId));
1306             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1307         }
1308         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1309                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId.longValue()), 5,
1310                 String.format("%s:%s", "ITM Flow Entry ", elanTag.toString()), 0, 0,
1311                 Uint64.valueOf(ITMConstants.COOKIE_ITM.toJava().add(BigInteger.valueOf(elanTag.longValue()))),
1312                 listMatchInfoBase, instructionInfos);
1313         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1314         LOG.info("Installed internal tunnel table (36) flow entry on dpn: {} for elan: {}, tag: {}", dpId,
1315                 elanInfo.getElanInstanceName(), elanTag);
1316     }
1317
1318     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId,
1319             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1320         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1321         if (etreeInstance != null) {
1322             setupTerminateServiceTable(elanInfo, dpId,
1323                 etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1324         }
1325     }
1326
1327     public void setupUnknownDMacTable(ElanInstance elanInfo, Uint64 dpId,
1328             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1329         long elanTag = elanInfo.getElanTag().toJava();
1330         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1331         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1332         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1333     }
1334
1335     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1336             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1337         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1338         if (etreeLeafTag != null) {
1339             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue().toJava();
1340             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1341             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1342         }
1343     }
1344
1345     private void installLocalUnknownFlow(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1346             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1347         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1348                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1349                 5, elanInfo.getElanInstanceName(), 0, 0,
1350                 Uint64.valueOf(ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.toJava().add(BigInteger.valueOf(elanTag))),
1351                 getMatchesForElanTag(elanTag, /* SH flag */false),
1352                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1353
1354         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1355         LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan: {}, tag: {}",
1356                 dpId, elanInfo.getElanInstanceName(), elanTag);
1357     }
1358
1359     private void installRemoteUnknownFlow(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1360             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1361         // only if ELAN can connect to external network, perform the following
1362
1363         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1364             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1365                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1366                     5, elanInfo.getElanInstanceName(), 0, 0,
1367                     Uint64.valueOf(ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.toJava().add(BigInteger.valueOf(elanTag))),
1368                     getMatchesForElanTag(elanTag, /* SH flag */true),
1369                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1370             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1371             LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan connected to "
1372                     + "external network: {}, tag: {}", dpId, elanInfo.getElanInstanceName(), elanTag);
1373         }
1374     }
1375
1376
1377     private void removeUnknownDmacFlow(Uint64 dpId, ElanInstance elanInfo,
1378             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1379             throws ExecutionException, InterruptedException {
1380         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1381                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1382         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1383
1384         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1385             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1386                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1387                     .build();
1388             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1389         }
1390     }
1391
1392     private void removeDefaultTermFlow(Uint64 dpId, long elanTag) {
1393         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1394     }
1395
1396     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1397             TypedWriteTransaction<Configuration> tx) {
1398         if (isStandardElanService(elanInterface)) {
1399             bindElanService(elanInfo.getElanTag().toJava(), elanInfo.getElanInstanceName(),
1400                     elanInterface.getName(), lportTag, tx);
1401         } else { // Etree service
1402             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1403         }
1404     }
1405
1406     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1407             TypedWriteTransaction<Configuration> tx) {
1408         int instructionKey = 0;
1409         List<Instruction> instructions = new ArrayList<>();
1410         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1411                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1412
1413         List<Action> actions = new ArrayList<>();
1414         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1415                 lportTag).buildAction());
1416         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1417                 elanTag).buildAction());
1418         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1419
1420         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1421                 ++instructionKey));
1422
1423         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1424         BoundServices serviceInfo = ElanUtils.getBoundServices(
1425                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1426                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1427         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1428         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1429                 bindServiceId);
1430         if (!existingElanService.isPresent()) {
1431             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1432             LOG.trace("Done binding elan service for elan: {} for interface: {}", elanInstanceName, interfaceName);
1433         }
1434     }
1435
1436     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1437             TypedWriteTransaction<Configuration> tx) {
1438         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1439             bindElanService(elanInfo.getElanTag().toJava(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1440                     lportTag, tx);
1441         } else {
1442             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1443             if (etreeInstance == null) {
1444                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1445                         elanInterface.getName(), elanInfo.getElanInstanceName());
1446             } else {
1447                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue().toJava(), elanInfo.getElanInstanceName(),
1448                         elanInterface.getName(), lportTag, tx);
1449             }
1450         }
1451     }
1452
1453     private static boolean isStandardElanService(ElanInterface elanInterface) {
1454         return elanInterface.augmentation(EtreeInterface.class) == null;
1455     }
1456
1457     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1458             throws ExecutionException, InterruptedException {
1459         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1460         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1461         if (tx.read(bindServiceId).get().isPresent()) {
1462             tx.delete(bindServiceId);
1463         }
1464     }
1465
1466     private static String getFlowRef(long tableId, long elanTag) {
1467         return String.valueOf(tableId) + elanTag;
1468     }
1469
1470     private static String getFlowRef(long tableId, long elanTag, String flowName) {
1471         return new StringBuilder().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1472                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1473     }
1474
1475     private static String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1476         return String.valueOf(tableId) + elanTag + shFlag;
1477     }
1478
1479     private static List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1480         List<Action> listAction = new ArrayList<>();
1481         int actionKey = 0;
1482         listAction.add(
1483             new ActionSetFieldTunnelId(Uint64.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1484         actionKey++;
1485         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1486         return listAction;
1487     }
1488
1489     private static DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, Uint64 dpId,
1490             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1491         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1492                 .withKey(new DpnInterfacesKey(dpId)).build();
1493         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1494                 CREATE_MISSING_PARENTS);
1495         LOG.trace("Updated operational dpn interfaces for elan: {} with interfaces: {}", elanInstanceName,
1496                 interfaceNames);
1497         return dpnInterface;
1498     }
1499
1500     /**
1501      * Delete elan dpn interface from operational DS.
1502      *
1503      * @param elanInstanceName
1504      *            the elan instance name
1505      * @param dpId
1506      *            the dp id
1507      */
1508     private static void deleteElanDpnInterface(String elanInstanceName, Uint64 dpId,
1509             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1510         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1511                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1512         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1513         if (dpnInterfaces.isPresent()) {
1514             tx.delete(dpnInterfacesId);
1515         }
1516     }
1517
1518     private static DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName,
1519             Uint64 dpId, TypedWriteTransaction<Operational> tx) {
1520         List<String> interfaceNames = new ArrayList<>();
1521         interfaceNames.add(interfaceName);
1522         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1523                 .withKey(new DpnInterfacesKey(dpId)).build();
1524         return dpnInterface;
1525     }
1526
1527     private static void createElanInterfaceTablesList(String interfaceName, TypedReadWriteTransaction<Operational> tx)
1528             throws ExecutionException, InterruptedException {
1529         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1530                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1531         Optional<ElanInterfaceMac> interfaceMacTables = tx.read(elanInterfaceMacTables).get();
1532         // Adding new Elan Interface Port to the operational DataStore without
1533         // Static-Mac Entries..
1534         if (!interfaceMacTables.isPresent()) {
1535             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1536                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1537             tx.put(ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1538                     CREATE_MISSING_PARENTS);
1539             LOG.trace("Created interface MAC table for interface: {}", interfaceName);
1540         }
1541     }
1542
1543     private static void createElanStateList(String elanInstanceName, String interfaceName,
1544             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1545         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1546         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1547         // Adding new Elan Interface Port to the operational DataStore without
1548         // Static-Mac Entries..
1549         if (elanInterfaceLists.isPresent()) {
1550             List<String> interfaceLists = new ArrayList<>();
1551             interfaceLists.add(interfaceName);
1552             List<String> existingInterfaceLists = elanInterfaceLists.get().getElanInterfaces();
1553             if (existingInterfaceLists != null && !existingInterfaceLists.isEmpty()) {
1554                 existingInterfaceLists.forEach(iface -> interfaceLists.add(iface));
1555             }
1556
1557             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1558                     .withKey(new ElanKey(elanInstanceName)).build();
1559             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1560             LOG.trace("Updated operational elan state for elan: {} with interfaces: {}", elanInstanceName,
1561                     interfaceLists);
1562         }
1563     }
1564
1565     private static boolean isOperational(InterfaceInfo interfaceInfo) {
1566         return interfaceInfo != null && interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1567     }
1568
1569     @SuppressWarnings("checkstyle:IllegalCatch")
1570     public void handleInternalTunnelStateEvent(Uint64 srcDpId, Uint64 dstDpId) {
1571         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1572         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1573                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1574         if (dpnInterfaceLists == null) {
1575             return;
1576         }
1577         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1578         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1579             int cnt = 0;
1580             String elanName = elanDpns.getElanInstanceName();
1581             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1582             if (elanInfo == null) {
1583                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1584                         + "skipping this ELAN for tunnel handling", elanName);
1585                 continue;
1586             }
1587             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1588                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1589                 continue;
1590             }
1591             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1592             if (dpnInterfaces == null) {
1593                 continue;
1594             }
1595             DpnInterfaces dstDpnIf = null;
1596             for (DpnInterfaces dpnIf : dpnInterfaces) {
1597                 Uint64 dpnIfDpId = dpnIf.getDpId();
1598                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1599                     cnt++;
1600                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1601                     cnt++;
1602                     dstDpnIf = dpnIf;
1603                 }
1604             }
1605             if (cnt == 2) {
1606                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1607                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1608                 jobCoordinator.enqueueJob(elanName, () -> {
1609                     // update Remote BC Group
1610                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1611                     try {
1612                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1613                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1614                                 confTx)).get();
1615                     } catch (RuntimeException e) {
1616                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1617                     }
1618                     Set<String> interfaceLists = new HashSet<>();
1619                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1620                     for (String ifName : interfaceLists) {
1621                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1622                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1623                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1624                             if (isOperational(interfaceInfo)) {
1625                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1626                             }
1627                             return emptyList();
1628                         }, ElanConstants.JOB_MAX_RETRIES);
1629                     }
1630                     return emptyList();
1631                 }, ElanConstants.JOB_MAX_RETRIES);
1632             }
1633
1634         }
1635     }
1636
1637     /**
1638      * Handle external tunnel state event.
1639      *
1640      * @param externalTunnel
1641      *            the external tunnel
1642      * @param intrf
1643      *            the interface
1644      */
1645     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1646         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1647             return;
1648         }
1649         // dpId/externalNodeId will be available either in source or destination
1650         // based on the tunnel end point
1651         Uint64 dpId = null;
1652         NodeId externalNodeId = null;
1653         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1654             dpId = Uint64.valueOf(externalTunnel.getSourceDevice());
1655             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1656         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1657             dpId = Uint64.valueOf(externalTunnel.getDestinationDevice());
1658             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1659         }
1660         if (dpId == null || externalNodeId == null) {
1661             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1662             return;
1663         }
1664
1665         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1666         if (dpnInterfaceLists == null) {
1667             return;
1668         }
1669         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1670         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1671             String elanName = elanDpns.getElanInstanceName();
1672             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1673
1674             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1675             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1676                     || dpnInterfaces.getInterfaces().isEmpty()) {
1677                 continue;
1678             }
1679             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1680
1681             final Uint64 finalDpId = dpId;
1682             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1683                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1684                 "Error setting up ELAN BGs");
1685             // install L2gwDevices local macs in dpn.
1686             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1687             // Install dpn macs on external device
1688             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1689                     externalNodeId);
1690         }
1691         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1692     }
1693
1694     /**
1695      * Installs dpn macs in external device. first it checks if the physical
1696      * locator towards this dpn tep is present or not if the physical locator is
1697      * present go ahead and add the ucast macs otherwise update the mcast mac
1698      * entry to include this dpn tep ip and schedule the job to put ucast macs
1699      * once the physical locator is programmed in device
1700      *
1701      * @param elanName
1702      *            the elan name
1703      * @param lstElanInterfaceNames
1704      *            the lst Elan interface names
1705      * @param dpnId
1706      *            the dpn id
1707      * @param externalNodeId
1708      *            the external node id
1709      */
1710     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, Uint64 dpnId,
1711             NodeId externalNodeId) {
1712         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1713                 externalNodeId.getValue());
1714         if (elanL2GwDevice == null) {
1715             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1716             return;
1717         }
1718         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1719         if (dpnTepIp == null) {
1720             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1721                     externalNodeId);
1722             return;
1723         }
1724
1725         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1726         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1727                 LogicalDatastoreType.OPERATIONAL);
1728         boolean phyLocAlreadyExists =
1729                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1730                 dpnTepIp);
1731         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1732                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1733         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1734
1735         if (phyLocAlreadyExists) {
1736             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1737             return;
1738         }
1739         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1740         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1741     }
1742
1743     /**
1744      * Validate external tunnel state event.
1745      *
1746      * @param externalTunnel
1747      *            the external tunnel
1748      * @param intrf
1749      *            the intrf
1750      * @return true, if successful
1751      */
1752     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1753         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1754             String srcDevice = externalTunnel.getDestinationDevice();
1755             String destDevice = externalTunnel.getSourceDevice();
1756             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1757                     LogicalDatastoreType.CONFIGURATION);
1758             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1759                     otherEndPointExtTunnel);
1760             if (otherEndPointExtTunnel != null) {
1761                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1762                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1763                 if (otherEndPointInterfaceOperational) {
1764                     return true;
1765                 } else {
1766                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1767                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1768                 }
1769             }
1770         }
1771         return false;
1772     }
1773
1774     private static List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1775         List<MatchInfo> mkMatches = new ArrayList<>();
1776         // Matching metadata
1777         mkMatches.add(
1778                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1779         mkMatches.add(new MatchTunnelId(Uint64.valueOf(lportTag)));
1780         return mkMatches;
1781     }
1782
1783     @Override
1784     protected ElanInterfaceManager getDataTreeChangeListener() {
1785         return this;
1786     }
1787
1788     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1789                                              Uint64 dpId) {
1790         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1791         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1792             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1793                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1794         try {
1795             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1796         } catch (InterruptedException e) {
1797             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1798         }
1799     }
1800 }