elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  15
  16 import com.google.common.base.Optional;
  17 import com.google.common.base.Preconditions;
  18 import com.google.common.collect.Lists;
  19 import com.google.common.util.concurrent.ListenableFuture;
  20 import java.math.BigInteger;
  21 import java.util.ArrayList;
  22 import java.util.Collections;
  23 import java.util.HashSet;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Objects;
  27 import java.util.Queue;
  28 import java.util.Set;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.ConcurrentLinkedQueue;
  31 import java.util.concurrent.ExecutionException;
  32 import java.util.concurrent.locks.ReentrantLock;
  33 import javax.annotation.Nullable;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.apache.commons.lang3.StringUtils;
  38 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.Datastore.Operational;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TransactionAdapter;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  50 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  51 import org.opendaylight.genius.itm.globals.ITMConstants;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  67 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  68 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.JvmGlobalLocks;
  72 import org.opendaylight.genius.utils.ServiceIndex;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  75 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  76 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  77 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  78 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  79 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  80 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  81 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  82 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  83 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  84 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  85 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  86 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  87 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  88 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  89 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  90 import org.opendaylight.serviceutils.srm.RecoverableListener;
  91 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  92 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  94 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 135 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 136 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 137 import org.slf4j.Logger;
 138 import org.slf4j.LoggerFactory;
 139
 140 /**
 141  * Class in charge of handling creations, modifications and removals of
 142  * ElanInterfaces.
 143  *
 144  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 145  */
 146 @Singleton
 147 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 148         implements RecoverableListener {
 149     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 150     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 151     private static final boolean SH_FLAG_SET = true;
 152     private static final boolean SH_FLAG_UNSET = false;
 153
 154     private final DataBroker broker;
 155     private final ManagedNewTransactionRunner txRunner;
 156     private final IMdsalApiManager mdsalManager;
 157     private final IInterfaceManager interfaceManager;
 158     private final IdManagerService idManager;
 159     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 160     private final INeutronVpnManager neutronVpnManager;
 161     private final ElanItmUtils elanItmUtils;
 162     private final ElanEtreeUtils elanEtreeUtils;
 163     private final ElanL2GatewayUtils elanL2GatewayUtils;
 164     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 165     private final ElanUtils elanUtils;
 166     private final JobCoordinator jobCoordinator;
 167     private final ElanInstanceCache elanInstanceCache;
 168     private final ElanInterfaceCache elanInterfaceCache;
 169
 170     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 171         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 172
 173     @Inject
 174     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 175                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 176                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 177                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 178                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 179                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 180                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 181                                 final ElanInstanceCache elanInstanceCache,
 182                                 final ElanInterfaceCache elanInterfaceCache,
 183                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 184                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 185         super(ElanInterface.class, ElanInterfaceManager.class);
 186         this.broker = dataBroker;
 187         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 188         this.idManager = managerService;
 189         this.mdsalManager = mdsalApiManager;
 190         this.interfaceManager = interfaceManager;
 191         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 192         this.neutronVpnManager = neutronVpnManager;
 193         this.elanItmUtils = elanItmUtils;
 194         this.elanEtreeUtils = elanEtreeUtils;
 195         this.elanL2GatewayUtils = elanL2GatewayUtils;
 196         this.elanUtils = elanUtils;
 197         this.jobCoordinator = jobCoordinator;
 198         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 199         this.elanInstanceCache = elanInstanceCache;
 200         this.elanInterfaceCache = elanInterfaceCache;
 201         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 202     }
 203
 204     @Override
 205     @PostConstruct
 206     public void init() {
 207         registerListener();
 208     }
 209
 210     @Override
 211     public void registerListener() {
 212         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 213     }
 214
 215     @Override
 216     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 217         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 218     }
 219
 220     @Override
 221     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 222         String interfaceName = del.getName();
 223         String elanInstanceName = del.getElanInstanceName();
 224         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 225         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 226             elanInterfaces.remove(del);
 227             if (elanInterfaces.isEmpty()) {
 228                 unProcessedElanInterfaces.remove(elanInstanceName);
 229             }
 230         }
 231         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 232         /*
 233          * Handling in case the elan instance is deleted.If the Elan instance is
 234          * deleted, there is no need to explicitly delete the elan interfaces
 235          */
 236         if (elanInfo == null) {
 237             return;
 238         }
 239         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 240         if (interfaceInfo == null && elanInfo.isExternal()) {
 241             // In deleting external network, the underlying ietf Inteface might have been removed
 242             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 243             // from Operational DS instead
 244             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 245         }
 246         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 247                 interfaceName, interfaceInfo, this);
 248         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 249     }
 250
 251     private static class RemoveElanInterfaceHolder {
 252         boolean isLastElanInterface = false;
 253         boolean isLastInterfaceOnDpn = false;
 254         BigInteger dpId = null;
 255     }
 256
 257     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 258     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 259             InterfaceInfo interfaceInfo) {
 260         String elanName = elanInfo.getElanInstanceName();
 261         long elanTag = elanInfo.getElanTag();
 262         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 263         List<ListenableFuture<Void>> futures = new ArrayList<>();
 264         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 265         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 266             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 267             if (elanState == null) {
 268                 return;
 269             }
 270             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 271                 @Nullable List<String> elanInterfaces = elanState.getElanInterfaces();
 272                 if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 273                     holder.isLastElanInterface = true;
 274                 }
 275                 if (interfaceInfo != null) {
 276                     holder.dpId = interfaceInfo.getDpId();
 277                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 278                         interfaceName, elanTag, interfaceTx);
 279                     /*
 280                      * If there are not elan ports, remove the unknown dmac, terminating
 281                      * service table flows, remote/local bc group
 282                      */
 283                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 284                         || dpnInterfaces.getInterfaces().isEmpty()) {
 285                         // No more Elan Interfaces in this DPN
 286                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 287                             holder.dpId);
 288                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 289                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag());
 290                         }
 291                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag());
 292                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 293                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 294                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 295                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 296                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 297                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 298                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 299                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 300                             }
 301                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 302                         }
 303                         holder.isLastInterfaceOnDpn = true;
 304                     } else {
 305                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 306                     }
 307                 }
 308             }));
 309         }));
 310         futures.forEach(ElanUtils::waitForTransactionToComplete);
 311
 312         if (holder.isLastInterfaceOnDpn && holder.dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 313             futures.add(
 314                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 315                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInfo, holder.dpId, confTx))));
 316         }
 317         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 318                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 319         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 320                 ElanConstants.JOB_MAX_RETRIES);
 321
 322         return futures;
 323     }
 324
 325     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 326             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 327             throws ExecutionException, InterruptedException {
 328         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 329         if (etreeLeafTag != null) {
 330             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 331             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 332         }
 333     }
 334
 335     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 336             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 337             throws ExecutionException, InterruptedException {
 338         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 339         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 340     }
 341
 342     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 343             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 344             throws ExecutionException, InterruptedException {
 345         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 346         if (etreeInstance != null) {
 347             BigInteger dpnId = interfaceInfo.getDpId();
 348             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 349             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 350             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 351         }
 352     }
 353
 354     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 355             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 356             throws ExecutionException, InterruptedException {
 357         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 358         if (etreeInstance != null) {
 359             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 360             BigInteger dpnId = interfaceInfo.getDpId();
 361             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 362             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 363             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 364         }
 365     }
 366
 367     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 368             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 369         String elanName = elanInfo.getElanInstanceName();
 370         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 371         if (elanState == null) {
 372             return elanState;
 373         }
 374         List<String> elanInterfaces = elanState.getElanInterfaces();
 375         boolean isRemoved = elanInterfaces != null && elanInterfaces.remove(interfaceName);
 376         if (!isRemoved) {
 377             return elanState;
 378         }
 379
 380         if (elanInterfaces.isEmpty()) {
 381             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 382             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 383             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 384         } else {
 385             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 386                     .withKey(new ElanKey(elanName)).build();
 387             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 388         }
 389         return elanState;
 390     }
 391
 392     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 393             throws ReadFailedException {
 394         // removing the ElanInterface from the config data_store if interface is
 395         // not present in Interface config DS
 396         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 397             interfaceName) == null
 398                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 399             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 400         }
 401     }
 402
 403     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 404             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 405         String elanName = elanInfo.getElanInstanceName();
 406         List<ListenableFuture<Void>> futures = new ArrayList<>();
 407         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 408             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 409                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 410                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 411                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 412                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 413                 if (interfaceInfo != null) {
 414                     if (existingElanInterfaceMac.isPresent()) {
 415                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 416                         if (existingMacEntries != null) {
 417                             List<PhysAddress> macAddresses = new ArrayList<>();
 418                             for (MacEntry macEntry : existingMacEntries) {
 419                                 PhysAddress macAddress = macEntry.getMacAddress();
 420                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 421                                         macAddress.getValue(), interfaceName);
 422                                 Optional<MacEntry> macEntryOptional =
 423                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 424                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 425                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 426                                 }
 427                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 428                                 macAddresses.add(macAddress);
 429                             }
 430
 431                             // Removing all those MACs from External Devices belonging
 432                             // to this ELAN
 433                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 434                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 435                             }
 436                         }
 437                     }
 438                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 439                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 440                 } else if (existingElanInterfaceMac.isPresent()) {
 441                     // Interface does not exist in ConfigDS, so lets remove everything
 442                     // about that interface related to Elan
 443                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 444                     if (macEntries != null) {
 445                         for (MacEntry macEntry : macEntries) {
 446                             PhysAddress macAddress = macEntry.getMacAddress();
 447                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 448                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 449                             }
 450                         }
 451                     }
 452                 }
 453                 if (existingElanInterfaceMac.isPresent()) {
 454                     interfaceTx.delete(elanInterfaceId);
 455                 }
 456                 unbindService(interfaceName, flowTx);
 457                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 458             }));
 459         }));
 460         return futures;
 461     }
 462
 463     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 464                                                                          String interfaceName, long elanTag,
 465                                                                          TypedReadWriteTransaction<Operational> tx)
 466             throws ExecutionException, InterruptedException {
 467         // FIXME: pass in and use ElanInstanceKey instead?
 468         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 469         lock.lock();
 470         try {
 471             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 472             if (dpnInterfaces != null) {
 473                 List<String> interfaceLists = dpnInterfaces.getInterfaces();
 474                 if (interfaceLists != null) {
 475                     interfaceLists.remove(interfaceName);
 476                 }
 477
 478                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 479                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 480                     deleteElanDpnInterface(elanName, dpId, tx);
 481                 } else {
 482                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 483                 }
 484             }
 485             return dpnInterfaces;
 486         } finally {
 487             lock.unlock();
 488         }
 489     }
 490
 491     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 492         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 493         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 494             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 495                 BigInteger currentDpId = dpnInterface.getDpId();
 496                 if (!currentDpId.equals(dpId) && dpnInterface.getInterfaces() != null) {
 497                     for (String elanInterface : dpnInterface.getInterfaces()) {
 498                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 499                         if (macs == null || macs.getMacEntry() == null) {
 500                             continue;
 501                         }
 502                         for (MacEntry mac : macs.getMacEntry()) {
 503                             removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 504                             removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 505                         }
 506                     }
 507                 }
 508             }
 509         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 510     }
 511
 512     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 513             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 514         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 515         if (etreeLeafTag != null) {
 516             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac, confTx);
 517         }
 518     }
 519
 520     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 521             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 522         mdsalManager
 523                 .removeFlow(confTx, dpId,
 524                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 525                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 526                                         mac.getMacAddress().getValue(), elanTag)));
 527     }
 528
 529     /*
 530     * Possible Scenarios for update
 531     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 532         then all {1,2,3,4} should be removed
 533
 534         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 535         then all {1,2,3,4} should be added
 536
 537         c. if orig = {1,2,3,4} updated={2,3,4}
 538         then 1 should be removed
 539
 540         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 541         then we should just add 5
 542
 543         e. if orig = {1,2,3,4} updated={2,3,4,5}
 544         then 1 should be removed , 5 should be added
 545     * */
 546     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 547     @Override
 548     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 549         // updating the static-Mac Entries for the existing elanInterface
 550         String elanName = update.getElanInstanceName();
 551         String interfaceName = update.getName();
 552
 553         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 554         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 555         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 556         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 557
 558         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 559                 deletedEntry.getMacAddress()));
 560
 561         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 562         * Hence add the macentries for the same.*/
 563         for (StaticMacEntries staticMacEntry : updatedEntries) {
 564             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 565                     staticMacEntry.getMacAddress());
 566             ListenableFutures.addErrorLogging(ElanUtils.waitForTransactionToComplete(
 567                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 568                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 569                     if (existingMacEntry.isPresent()) {
 570                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 571                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 572                             tx);
 573                     } else {
 574                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 575                             elanName, interfaceName, staticMacEntry, tx);
 576                     }
 577                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 578         }
 579     }
 580
 581     @Override
 582     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 583         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 584             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 585             String interfaceName = elanInterfaceAdded.getName();
 586             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 587             if (interfaceInfo == null) {
 588                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 589                 return;
 590             }
 591             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 592
 593             if (elanInstance == null) {
 594                 // Add the ElanInstance in the Configuration data-store
 595                 List<String> elanInterfaces = new ArrayList<>();
 596                 elanInterfaces.add(interfaceName);
 597                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 598                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 599                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 600                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 601             }
 602
 603             Long elanTag = elanInstance.getElanTag();
 604             // If elan tag is not updated, then put the elan interface into
 605             // unprocessed entry map and entry. Let entries
 606             // in this map get processed during ELAN update DCN.
 607             if (elanTag == null || elanTag == 0L) {
 608                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 609                 if (elanInterfaces == null) {
 610                     elanInterfaces = new ConcurrentLinkedQueue<>();
 611                 }
 612                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 613                     elanInterfaces.add(elanInterfaceAdded);
 614                 }
 615                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 616                 return;
 617             }
 618             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 619                     interfaceInfo, elanInstance, this);
 620             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 621         }), LOG, "Error processing added ELAN interface");
 622     }
 623
 624     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 625         List<ListenableFuture<Void>> futures = new ArrayList<>();
 626         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 627         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 628             return futures;
 629         }
 630         for (ElanInterface elanInterface : elanInterfaces) {
 631             String interfaceName = elanInterface.getName();
 632             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 633             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 634         }
 635         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 636         return futures;
 637     }
 638
 639     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 640             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 641         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 642                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 643         List<DpnInterfaces> dpnInterfaceLists = null;
 644         if (elanDpnInterfacesList != null) {
 645             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 646         }
 647         if (dpnInterfaceLists == null) {
 648             dpnInterfaceLists = new ArrayList<>();
 649         }
 650         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 651             BigInteger dstDpId = interfaceInfo.getDpId();
 652             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 653                 continue;
 654             }
 655             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 656             for (String remoteIf : remoteElanInterfaces) {
 657                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 658                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 659                 if (elanIfMac == null || remoteInterface == null) {
 660                     continue;
 661                 }
 662                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 663                 if (remoteMacEntries != null) {
 664                     for (MacEntry macEntry : remoteMacEntries) {
 665                         String macAddress = macEntry.getMacAddress().getValue();
 666                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 667                                 dstDpId, elanInstance.getElanInstanceName());
 668                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 669                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 670                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 671                     }
 672                 }
 673             }
 674         }
 675     }
 676
 677     private static class AddElanInterfaceHolder {
 678         private DpnInterfaces dpnInterfaces = null;
 679         private boolean isFirstInterfaceInDpn = false;
 680         private BigInteger dpId;
 681     }
 682
 683     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 684     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 685             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 686         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 687         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 688         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 689
 690         String interfaceName = elanInterface.getName();
 691         String elanInstanceName = elanInterface.getElanInstanceName();
 692
 693         List<ListenableFuture<Void>> futures = new ArrayList<>();
 694         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 695         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 696             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 697             if (elanInfo == null) {
 698                 List<String> elanInterfaces = new ArrayList<>();
 699                 elanInterfaces.add(interfaceName);
 700                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 701                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 702                         operTx)));
 703             } else {
 704                 createElanStateList(elanInstanceName, interfaceName, operTx);
 705             }
 706             // Specific actions to the DPN where the ElanInterface has been added,
 707             // for example, programming the
 708             // External tunnel table if needed or adding the ElanInterface to the
 709             // DpnInterfaces in the operational DS.
 710             holder.dpId = interfaceInfo.getDpId();
 711             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 712                 // FIXME: use elanInstaince.key() instead?
 713                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 714                 lock.lock();
 715                 try {
 716                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 717                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 718                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 719                     if (ElanUtils.isVlan(elanInstance)) {
 720                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 721                             elanInstanceName, existingElanDpnInterfaces);
 722                     } else {
 723                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 724                     }
 725                     if (holder.isFirstInterfaceInDpn) {
 726                         // ELAN's 1st ElanInterface added to this DPN
 727                         if (!existingElanDpnInterfaces.isPresent()) {
 728                             holder.dpnInterfaces =
 729                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 730                         } else {
 731                             @Nullable List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 732                             List<String> elanInterfaces =
 733                                 existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 734                             elanInterfaces.add(interfaceName);
 735                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 736                                 elanInterfaces, operTx);
 737                         }
 738                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 739                         // table, but only if Elan has VNI
 740                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 741                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 742                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 743                         }
 744                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 745                             interfaceName);
 746                     } else {
 747                         @Nullable List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 748                         List<String> elanInterfaces =
 749                             existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 750                         elanInterfaces.add(interfaceName);
 751                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 752                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 753                                 interfaceName);
 754                         }
 755                         holder.dpnInterfaces =
 756                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 757                     }
 758                 } finally {
 759                     lock.unlock();
 760                 }
 761             }
 762
 763             // add code to install Local/Remote BC group, unknow DMAC entry,
 764             // terminating service table flow entry
 765             // call bindservice of interfacemanager to create ingress table flow
 766             // enty.
 767             // Add interface to the ElanInterfaceForwardingEntires Container
 768             createElanInterfaceTablesList(interfaceName, operTx);
 769         }));
 770         futures.forEach(ElanUtils::waitForTransactionToComplete);
 771         futures.add(
 772             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 773                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 774                     holder.isFirstInterfaceInDpn, confTx))));
 775
 776         // add the vlan provider interface to remote BC group for the elan
 777         // for internal vlan networks
 778         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 779             if (interfaceManager.isExternalInterface(interfaceName)) {
 780                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 781                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 782             }
 783         }
 784
 785         if (holder.isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 786             //update the remote-DPNs remoteBC group entry with Tunnels
 787             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance,
 788                 holder.dpId);
 789             futures.add(
 790                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 791                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInstance, holder.dpId, confTx))));
 792         }
 793
 794         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 795         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 796                 elanInterface, interfaceInfo, elanInstance, holder.isFirstInterfaceInDpn, this);
 797         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 798         return futures;
 799     }
 800
 801     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 802     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 803             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 804         String elanInstanceName = elanInstance.getElanInstanceName();
 805         String interfaceName = elanInterface.getName();
 806         List<ListenableFuture<Void>> futures = new ArrayList<>();
 807         BigInteger dpId = interfaceInfo.getDpId();
 808         boolean isInterfaceOperational = isOperational(interfaceInfo);
 809         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 810             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 811                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 812                     isFirstInterfaceInDpn, confTx, operTx);
 813
 814                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 815                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 816
 817                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 818                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 819                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 820                             staticMacEntry.getMacAddress());
 821                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 822                             LogicalDatastoreType.OPERATIONAL, macId);
 823                         if (existingMacEntry.isPresent()) {
 824                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 825                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 826                                 existingMacEntry.get(), operTx);
 827                         } else {
 828                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 829                                 interfaceName, staticMacEntry, operTx);
 830                         }
 831
 832                         if (isInterfaceOperational) {
 833                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 834                             // DPNs
 835                             String macAddress = staticMacEntry.getMacAddress().getValue();
 836                             LOG.info(
 837                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 838                                     + " {}",
 839                                 macAddress, elanInstanceName, interfaceName);
 840                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 841                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 842                         }
 843                     }
 844
 845                     if (isInterfaceOperational) {
 846                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 847                         // on purpose.
 848                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 849                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 850                         }
 851                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 852                             staticMacAddresses);
 853                     }
 854                 }
 855             }));
 856         }));
 857         futures.forEach(ElanUtils::waitForTransactionToComplete);
 858         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 859             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 860             try {
 861                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 862                 if (neutronPort != null) {
 863                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 864                 }
 865             } catch (IllegalArgumentException ex) {
 866                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 867             }
 868         }
 869         return futures;
 870     }
 871
 872     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 873             PhysAddress physAddress) {
 874         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 875         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 876         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 877                 LogicalDatastoreType.OPERATIONAL, macId);
 878
 879         if (!existingMacEntry.isPresent()) {
 880             return;
 881         }
 882
 883         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 884                 .withKey(new MacEntryKey(physAddress)).build();
 885         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 886                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 887     }
 888
 889     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 890             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 891         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 892         if (!existingElanDpnInterfaces.isPresent()) {
 893             return true;
 894         }
 895         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 896             return false;
 897         }
 898         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 899         int dummyInterfaceCount =  0;
 900         List<String> interfaces = dpnInterfaces.getInterfaces();
 901         if (interfaces == null) {
 902             return true;
 903         }
 904         if (interfaces.contains(routerPortUuid)) {
 905             dummyInterfaceCount++;
 906         }
 907         if (interfaces.contains(elanInstanceName)) {
 908             dummyInterfaceCount++;
 909         }
 910         return interfaces.size() == dummyInterfaceCount;
 911     }
 912
 913     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 914         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 915                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 916     }
 917
 918     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 919             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx,
 920             TypedWriteTransaction<Operational> operTx) {
 921         if (!isOperational(interfaceInfo)) {
 922             return;
 923         }
 924         BigInteger dpId = interfaceInfo.getDpId();
 925         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 926             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 927         }
 928         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 929         if (isFirstInterfaceInDpn) {
 930             // Terminating Service , UnknownDMAC Table.
 931             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 932             // for ELAN Instance is VxLAN
 933             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 934                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 935             }
 936             setupUnknownDMacTable(elanInstance, dpId, confTx);
 937             /*
 938              * Install remote DMAC flow. This is required since this DPN is
 939              * added later to the elan instance and remote DMACs of other
 940              * interfaces in this elan instance are not present in the current
 941              * dpn.
 942              */
 943             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 944                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 945                         elanInstance.getElanInstanceName());
 946                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 947             }
 948         }
 949         // bind the Elan service to the Interface
 950         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 951     }
 952
 953     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 954             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 955         if (!isOperational(interfaceInfo)) {
 956             return;
 957         }
 958         // LocalBroadcast Group creation with elan-Interfaces
 959         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
 960         if (isFirstInterfaceInDpn) {
 961             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 962             BigInteger dpId = interfaceInfo.getDpId();
 963             // RemoteBroadcast Group creation
 964             try {
 965                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 966             } catch (InterruptedException e1) {
 967                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 968             }
 969             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId, confTx);
 970             try {
 971                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 972             } catch (InterruptedException e1) {
 973                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 974             }
 975         }
 976     }
 977
 978     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 979             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 980         int ifTag = interfaceInfo.getInterfaceTag();
 981         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 982                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 983                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 984                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 985                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 986
 987         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
 988
 989         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 990                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 991                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 992                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 993
 994         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
 995     }
 996
 997     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 998             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
 999         int ifTag = interfaceInfo.getInterfaceTag();
1000         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1001             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
1002
1003         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1004
1005         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1006             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
1007             0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
1008             getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1009
1010         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1011     }
1012
1013     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId,
1014             TypedWriteTransaction<Configuration> confTx) {
1015         int elanTag = elanInfo.getElanTag().intValue();
1016         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
1017         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId, confTx);
1018         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1019         if (etreeInstance != null) {
1020             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
1021             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
1022             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId, confTx);
1023         }
1024     }
1025
1026     @SuppressWarnings("checkstyle:IllegalCatch")
1027     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId,
1028             TypedWriteTransaction<Configuration> confTx) {
1029         int bucketId = 0;
1030         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
1031         if (elanDpns != null) {
1032             List<DpnInterfaces> dpnInterfaces = elanDpns.nonnullDpnInterfaces();
1033             for (DpnInterfaces dpnInterface : dpnInterfaces) {
1034                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
1035                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
1036                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
1037                     List<Action> listAction = new ArrayList<>();
1038                     int actionKey = 0;
1039                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1040                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
1041                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1042                     bucketId++;
1043                     for (DpnInterfaces otherFes : dpnInterfaces) {
1044                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
1045                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
1046                             && !otherFes.getInterfaces().isEmpty()) {
1047                             try {
1048                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
1049                                         dpnInterface.getDpId(), otherFes.getDpId(),
1050                                         elanUtils.isOpenstackVniSemanticsEnforced()
1051                                                 ? ElanUtils.getVxlanSegmentationId(elanInfo) : elanTag);
1052                                 if (!remoteListActionInfo.isEmpty()) {
1053                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
1054                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1055                                     bucketId++;
1056                                 }
1057                             } catch (Exception ex) {
1058                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1059                                         + "Logical Group Interface not found between source Dpn - {}, "
1060                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1061                                 return;
1062                             }
1063                         }
1064                     }
1065                     List<Bucket> elanL2GwDevicesBuckets = elanL2GatewayMulticastUtils
1066                             .getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnInterface.getDpId(), bucketId);
1067                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1068
1069                     if (remoteListBucketInfo.isEmpty()) {
1070                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1071                         continue;
1072                     }
1073                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1074                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1075                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1076                     mdsalManager.addGroup(confTx, dpnInterface.getDpId(), group);
1077                 }
1078             }
1079             try {
1080                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1081             } catch (InterruptedException e1) {
1082                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1083             }
1084         }
1085     }
1086
1087     private List<MatchInfo> buildMatchesForVni(Long vni) {
1088         List<MatchInfo> mkMatches = new ArrayList<>();
1089         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1090         mkMatches.add(match);
1091         return mkMatches;
1092     }
1093
1094     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1095         List<InstructionInfo> mkInstructions = new ArrayList<>();
1096         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1097         return mkInstructions;
1098     }
1099
1100     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1101         List<MatchInfo> mkMatches = new ArrayList<>();
1102         // Matching metadata
1103         mkMatches.add(new MatchMetadata(
1104                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1105         return mkMatches;
1106     }
1107
1108     /**
1109      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1110      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1111      *
1112      * @param elanTag
1113      *            elanTag to be written in metadata when flow is selected
1114      * @return the instructions ready to be installed in a flow
1115      */
1116     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1117         List<InstructionInfo> mkInstructions = new ArrayList<>();
1118         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1119                 .getElanMetadataMask()));
1120         /* applicable for EXTERNAL_TUNNEL_TABLE only
1121         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1122         */
1123         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1124         return mkInstructions;
1125     }
1126
1127     // Install DMAC entry on dst DPN
1128     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1129             BigInteger dstDpId) {
1130         String interfaceName = interfaceInfo.getInterfaceName();
1131         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1132         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1133             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1134             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1135                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1136                     for (MacEntry macEntry : macEntries) {
1137                         String macAddress = macEntry.getMacAddress().getValue();
1138                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1139                             interfaceName);
1140                         synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1141                             interfaceInfo.getDpId())) {
1142                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1143                                 macAddress);
1144                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1145                         }
1146                     }
1147                 })));
1148         }
1149         return emptyList();
1150     }
1151
1152     private void createDropBucket(List<Bucket> listBucket) {
1153         List<Action> actionsInfos = new ArrayList<>();
1154         actionsInfos.add(new ActionDrop().buildAction());
1155         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1156                 MDSALUtil.WATCH_GROUP);
1157         listBucket.add(dropBucket);
1158     }
1159
1160     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1161             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1162         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1163         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1164     }
1165
1166     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1167             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1168         List<Bucket> listBucket = new ArrayList<>();
1169         int bucketId = 0;
1170         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1171
1172         List<String> interfaces = new ArrayList<>();
1173         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1174             interfaces = newDpnInterface.getInterfaces();
1175         }
1176         for (String ifName : interfaces) {
1177             // In case if there is a InterfacePort in the cache which is not in
1178             // operational state, skip processing it
1179             InterfaceInfo ifInfo = interfaceManager
1180                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1181             if (!isOperational(ifInfo)) {
1182                 continue;
1183             }
1184
1185             if (!interfaceManager.isExternalInterface(ifName)) {
1186                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1187                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1188                 bucketId++;
1189             }
1190         }
1191
1192         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1193                 MDSALUtil.buildBucketLists(listBucket));
1194         LOG.trace("installing the localBroadCast Group:{}", group);
1195         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1196     }
1197
1198     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1199             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1200         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1201         if (etreeInstance != null) {
1202             List<Bucket> listBucket = new ArrayList<>();
1203             int bucketId = 0;
1204
1205             List<String> interfaces = new ArrayList<>();
1206             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1207                 interfaces = newDpnInterface.getInterfaces();
1208             }
1209             for (String ifName : interfaces) {
1210                 // In case if there is a InterfacePort in the cache which is not
1211                 // in
1212                 // operational state, skip processing it
1213                 InterfaceInfo ifInfo = interfaceManager
1214                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1215                 if (!isOperational(ifInfo)) {
1216                     continue;
1217                 }
1218
1219                 if (!interfaceManager.isExternalInterface(ifName)) {
1220                     // only add root interfaces
1221                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1222                 }
1223             }
1224
1225             if (listBucket.isEmpty()) { // No Buckets
1226                 createDropBucket(listBucket);
1227             }
1228
1229             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1230             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1231             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1232                     MDSALUtil.buildBucketLists(listBucket));
1233             LOG.trace("installing the localBroadCast Group:{}", group);
1234             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1235         }
1236     }
1237
1238     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1239             InterfaceInfo ifInfo) {
1240         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1241         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1242             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1243                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1244             bucketId++;
1245         }
1246         return bucketId;
1247     }
1248
1249     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1250             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1251             throws ExecutionException, InterruptedException {
1252         BigInteger dpnId = interfaceInfo.getDpId();
1253         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1254         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1255         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1256     }
1257
1258     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1259             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1260             throws ExecutionException, InterruptedException {
1261         BigInteger dpnId = interfaceInfo.getDpId();
1262         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1263         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1264         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1265     }
1266
1267     /**
1268      * Installs a flow in the External Tunnel table consisting in translating
1269      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1270      * elanTag that will be used later in the ELANs pipeline.
1271      * @param dpnId the dpn id
1272      */
1273     private void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1274             TypedWriteTransaction<Configuration> confTx) {
1275         long elanTag = elanInfo.getElanTag();
1276         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1277                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1278                 elanInfo.getElanInstanceName(), // flowName
1279                 0, // idleTimeout
1280                 0, // hardTimeout
1281                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1282                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1283                 getInstructionsIntOrExtTunnelTable(elanTag));
1284
1285         mdsalManager.addFlow(confTx, flowEntity);
1286     }
1287
1288     /**
1289      * Removes, from External Tunnel table, the flow that translates from VNI to
1290      * elanTag. Important: ensure this method is only called whenever there is
1291      * no other ElanInterface in the specified DPN
1292      *
1293      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1294      */
1295     private void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1296             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1297         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1298         // last ElanInstance plus
1299         // adding a new one does (almost at the same time) are executed in that
1300         // exact order
1301
1302         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1303         FlowEntity flowEntity = new FlowEntityBuilder()
1304             .setDpnId(dpnId)
1305             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1306             .setFlowId(flowId)
1307             .build();
1308         mdsalManager.removeFlow(confTx, flowEntity);
1309     }
1310
1311     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1312             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1313         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1314         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1315     }
1316
1317     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1318             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1319         List<? extends MatchInfoBase> listMatchInfoBase;
1320         List<InstructionInfo> instructionInfos;
1321         long serviceId;
1322         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1323             serviceId = elanTag;
1324             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1325             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1326         } else {
1327             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1328             listMatchInfoBase = buildMatchesForVni(serviceId);
1329             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1330         }
1331         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1332                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1333                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1334                 instructionInfos);
1335         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1336     }
1337
1338     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1339             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1340         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1341         if (etreeInstance != null) {
1342             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1343         }
1344     }
1345
1346     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId,
1347             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1348         long elanTag = elanInfo.getElanTag();
1349         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1350         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1351         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1352     }
1353
1354     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1355             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1356         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1357         if (etreeLeafTag != null) {
1358             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1359             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1360             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1361         }
1362     }
1363
1364     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1365             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1366         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1367                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1368                 5, elanInfo.getElanInstanceName(), 0, 0,
1369                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1370                 getMatchesForElanTag(elanTag, /* SH flag */false),
1371                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1372
1373         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1374     }
1375
1376     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1377             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1378         // only if ELAN can connect to external network, perform the following
1379
1380         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1381             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1382                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1383                     5, elanInfo.getElanInstanceName(), 0, 0,
1384                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1385                     getMatchesForElanTag(elanTag, /* SH flag */true),
1386                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1387             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1388         }
1389     }
1390
1391
1392     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
1393             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1394             throws ExecutionException, InterruptedException {
1395         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1396                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1397         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1398
1399         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1400             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1401                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1402                     .build();
1403             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1404         }
1405     }
1406
1407     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1408         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1409     }
1410
1411     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1412             TypedWriteTransaction<Configuration> tx) {
1413         if (isStandardElanService(elanInterface)) {
1414             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1415                     elanInterface.getName(), lportTag, tx);
1416         } else { // Etree service
1417             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1418         }
1419     }
1420
1421     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1422             TypedWriteTransaction<Configuration> tx) {
1423         int instructionKey = 0;
1424         List<Instruction> instructions = new ArrayList<>();
1425         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1426                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1427
1428         List<Action> actions = new ArrayList<>();
1429         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1430                 lportTag).buildAction());
1431         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1432                 elanTag).buildAction());
1433         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1434
1435         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1436                 ++instructionKey));
1437
1438         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1439         BoundServices serviceInfo = ElanUtils.getBoundServices(
1440                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1441                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1442         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1443         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1444                 bindServiceId);
1445         if (!existingElanService.isPresent()) {
1446             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1447         }
1448     }
1449
1450     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1451             TypedWriteTransaction<Configuration> tx) {
1452         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1453             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1454                     lportTag, tx);
1455         } else {
1456             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1457             if (etreeInstance == null) {
1458                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1459                         elanInterface.getName(), elanInfo.getElanInstanceName());
1460             } else {
1461                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1462                         elanInterface.getName(), lportTag, tx);
1463             }
1464         }
1465     }
1466
1467     private boolean isStandardElanService(ElanInterface elanInterface) {
1468         return elanInterface.augmentation(EtreeInterface.class) == null;
1469     }
1470
1471     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1472             throws ExecutionException, InterruptedException {
1473         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1474         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1475         if (tx.read(bindServiceId).get().isPresent()) {
1476             tx.delete(bindServiceId);
1477         }
1478     }
1479
1480     private String getFlowRef(long tableId, long elanTag) {
1481         return String.valueOf(tableId) + elanTag;
1482     }
1483
1484     private String getFlowRef(long tableId, long elanTag, String flowName) {
1485         return new StringBuffer().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1486                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1487     }
1488
1489     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1490         return String.valueOf(tableId) + elanTag + shFlag;
1491     }
1492
1493     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1494         List<Action> listAction = new ArrayList<>();
1495         int actionKey = 0;
1496         listAction.add(
1497             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1498         actionKey++;
1499         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1500         return listAction;
1501     }
1502
1503     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1504             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1505         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1506                 .withKey(new DpnInterfacesKey(dpId)).build();
1507         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1508                 CREATE_MISSING_PARENTS);
1509         return dpnInterface;
1510     }
1511
1512     /**
1513      * Delete elan dpn interface from operational DS.
1514      *
1515      * @param elanInstanceName
1516      *            the elan instance name
1517      * @param dpId
1518      *            the dp id
1519      */
1520     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId,
1521             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1522         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1523                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1524         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1525         if (dpnInterfaces.isPresent()) {
1526             tx.delete(dpnInterfacesId);
1527         }
1528     }
1529
1530     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1531             TypedWriteTransaction<Operational> tx) {
1532         List<String> interfaceNames = new ArrayList<>();
1533         interfaceNames.add(interfaceName);
1534         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1535                 .withKey(new DpnInterfacesKey(dpId)).build();
1536         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1537                 CREATE_MISSING_PARENTS);
1538         return dpnInterface;
1539     }
1540
1541     private void createElanInterfaceTablesList(String interfaceName, TypedReadWriteTransaction<Operational> tx)
1542             throws ExecutionException, InterruptedException {
1543         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1544                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1545         Optional<ElanInterfaceMac> interfaceMacTables = tx.read(elanInterfaceMacTables).get();
1546         // Adding new Elan Interface Port to the operational DataStore without
1547         // Static-Mac Entries..
1548         if (!interfaceMacTables.isPresent()) {
1549             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1550                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1551             tx.put(ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1552                     CREATE_MISSING_PARENTS);
1553         }
1554     }
1555
1556     private void createElanStateList(String elanInstanceName, String interfaceName,
1557             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1558         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1559         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1560         // Adding new Elan Interface Port to the operational DataStore without
1561         // Static-Mac Entries..
1562         if (elanInterfaceLists.isPresent()) {
1563             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1564             if (interfaceLists == null) {
1565                 interfaceLists = new ArrayList<>();
1566             }
1567             interfaceLists.add(interfaceName);
1568             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1569                     .withKey(new ElanKey(elanInstanceName)).build();
1570             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1571         }
1572     }
1573
1574     private boolean isOperational(InterfaceInfo interfaceInfo) {
1575         if (interfaceInfo == null) {
1576             return false;
1577         }
1578         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1579     }
1580
1581     @SuppressWarnings("checkstyle:IllegalCatch")
1582     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) {
1583         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1584         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1585                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1586         if (dpnInterfaceLists == null) {
1587             return;
1588         }
1589         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1590         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1591             int cnt = 0;
1592             String elanName = elanDpns.getElanInstanceName();
1593             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1594             if (elanInfo == null) {
1595                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1596                         + "skipping this ELAN for tunnel handling", elanName);
1597                 continue;
1598             }
1599             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1600                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1601                 continue;
1602             }
1603             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1604             if (dpnInterfaces == null) {
1605                 continue;
1606             }
1607             DpnInterfaces dstDpnIf = null;
1608             for (DpnInterfaces dpnIf : dpnInterfaces) {
1609                 BigInteger dpnIfDpId = dpnIf.getDpId();
1610                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1611                     cnt++;
1612                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1613                     cnt++;
1614                     dstDpnIf = dpnIf;
1615                 }
1616             }
1617             if (cnt == 2) {
1618                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1619                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1620                 jobCoordinator.enqueueJob(elanName, () -> {
1621                     // update Remote BC Group
1622                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1623                     try {
1624                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1625                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1626                                 confTx)).get();
1627                     } catch (RuntimeException e) {
1628                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1629                     }
1630                     Set<String> interfaceLists = new HashSet<>();
1631                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1632                     for (String ifName : interfaceLists) {
1633                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1634                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1635                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1636                             if (isOperational(interfaceInfo)) {
1637                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1638                             }
1639                             return emptyList();
1640                         }, ElanConstants.JOB_MAX_RETRIES);
1641                     }
1642                     return emptyList();
1643                 }, ElanConstants.JOB_MAX_RETRIES);
1644             }
1645
1646         }
1647     }
1648
1649     /**
1650      * Handle external tunnel state event.
1651      *
1652      * @param externalTunnel
1653      *            the external tunnel
1654      * @param intrf
1655      *            the interface
1656      */
1657     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1658         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1659             return;
1660         }
1661         // dpId/externalNodeId will be available either in source or destination
1662         // based on the tunnel end point
1663         BigInteger dpId = null;
1664         NodeId externalNodeId = null;
1665         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1666             dpId = new BigInteger(externalTunnel.getSourceDevice());
1667             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1668         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1669             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1670             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1671         }
1672         if (dpId == null || externalNodeId == null) {
1673             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1674             return;
1675         }
1676
1677         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1678         if (dpnInterfaceLists == null) {
1679             return;
1680         }
1681         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1682         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1683             String elanName = elanDpns.getElanInstanceName();
1684             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1685
1686             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1687             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1688                     || dpnInterfaces.getInterfaces().isEmpty()) {
1689                 continue;
1690             }
1691             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1692
1693             final BigInteger finalDpId = dpId;
1694             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1695                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1696                 "Error setting up ELAN BGs");
1697             // install L2gwDevices local macs in dpn.
1698             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1699             // Install dpn macs on external device
1700             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1701                     externalNodeId);
1702         }
1703         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1704     }
1705
1706     /**
1707      * Installs dpn macs in external device. first it checks if the physical
1708      * locator towards this dpn tep is present or not if the physical locator is
1709      * present go ahead and add the ucast macs otherwise update the mcast mac
1710      * entry to include this dpn tep ip and schedule the job to put ucast macs
1711      * once the physical locator is programmed in device
1712      *
1713      * @param elanName
1714      *            the elan name
1715      * @param lstElanInterfaceNames
1716      *            the lst Elan interface names
1717      * @param dpnId
1718      *            the dpn id
1719      * @param externalNodeId
1720      *            the external node id
1721      */
1722     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1723             NodeId externalNodeId) {
1724         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1725                 externalNodeId.getValue());
1726         if (elanL2GwDevice == null) {
1727             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1728             return;
1729         }
1730         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1731         if (dpnTepIp == null) {
1732             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1733                     externalNodeId);
1734             return;
1735         }
1736
1737         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1738         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1739                 LogicalDatastoreType.OPERATIONAL);
1740         boolean phyLocAlreadyExists =
1741                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1742                 dpnTepIp);
1743         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1744                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1745         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1746
1747         if (phyLocAlreadyExists) {
1748             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1749             return;
1750         }
1751         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1752         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1753     }
1754
1755     /**
1756      * Validate external tunnel state event.
1757      *
1758      * @param externalTunnel
1759      *            the external tunnel
1760      * @param intrf
1761      *            the intrf
1762      * @return true, if successful
1763      */
1764     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1765         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1766             String srcDevice = externalTunnel.getDestinationDevice();
1767             String destDevice = externalTunnel.getSourceDevice();
1768             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1769                     LogicalDatastoreType.CONFIGURATION);
1770             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1771                     otherEndPointExtTunnel);
1772             if (otherEndPointExtTunnel != null) {
1773                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1774                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1775                 if (otherEndPointInterfaceOperational) {
1776                     return true;
1777                 } else {
1778                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1779                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1780                 }
1781             }
1782         }
1783         return false;
1784     }
1785
1786     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1787         List<MatchInfo> mkMatches = new ArrayList<>();
1788         // Matching metadata
1789         mkMatches.add(
1790                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1791         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1792         return mkMatches;
1793     }
1794
1795     @Override
1796     protected ElanInterfaceManager getDataTreeChangeListener() {
1797         return this;
1798     }
1799
1800     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1801                                              BigInteger dpId) {
1802         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1803         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1804             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1805                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1806         try {
1807             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1808         } catch (InterruptedException e) {
1809             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1810         }
1811     }
1812 }