elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  15 import static org.opendaylight.netvirt.elan.utils.ElanUtils.requireNonNullElse;
  16
  17 import com.google.common.base.Optional;
  18 import com.google.common.base.Preconditions;
  19 import com.google.common.collect.Lists;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import java.math.BigInteger;
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Queue;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import javax.annotation.PostConstruct;
  34 import javax.inject.Inject;
  35 import javax.inject.Singleton;
  36 import org.apache.commons.lang3.StringUtils;
  37 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  38 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  39 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  40 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  41 import org.opendaylight.genius.infra.Datastore.Configuration;
  42 import org.opendaylight.genius.infra.Datastore.Operational;
  43 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  45 import org.opendaylight.genius.infra.TransactionAdapter;
  46 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  47 import org.opendaylight.genius.infra.TypedWriteTransaction;
  48 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  49 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  50 import org.opendaylight.genius.itm.globals.ITMConstants;
  51 import org.opendaylight.genius.mdsalutil.FlowEntity;
  52 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  53 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  54 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  55 import org.opendaylight.genius.mdsalutil.MatchInfo;
  56 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  57 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  64 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  67 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  70 import org.opendaylight.genius.utils.ServiceIndex;
  71 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  72 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  73 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  74 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  75 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  76 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  77 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  78 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  79 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  80 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  81 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  82 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  83 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  84 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  85 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  86 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  87 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  88 import org.opendaylight.serviceutils.srm.RecoverableListener;
  89 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  90 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  91 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  92 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 133 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 134 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 135 import org.slf4j.Logger;
 136 import org.slf4j.LoggerFactory;
 137
 138 /**
 139  * Class in charge of handling creations, modifications and removals of
 140  * ElanInterfaces.
 141  *
 142  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 143  */
 144 @Singleton
 145 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 146         implements RecoverableListener {
 147     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 148     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 149     private static final boolean SH_FLAG_SET = true;
 150     private static final boolean SH_FLAG_UNSET = false;
 151
 152     private final DataBroker broker;
 153     private final ManagedNewTransactionRunner txRunner;
 154     private final IMdsalApiManager mdsalManager;
 155     private final IInterfaceManager interfaceManager;
 156     private final IdManagerService idManager;
 157     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 158     private final INeutronVpnManager neutronVpnManager;
 159     private final ElanItmUtils elanItmUtils;
 160     private final ElanEtreeUtils elanEtreeUtils;
 161     private final ElanL2GatewayUtils elanL2GatewayUtils;
 162     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 163     private final ElanUtils elanUtils;
 164     private final JobCoordinator jobCoordinator;
 165     private final ElanInstanceCache elanInstanceCache;
 166     private final ElanInterfaceCache elanInterfaceCache;
 167
 168     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 169         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 170
 171     @Inject
 172     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 173                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 174                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 175                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 176                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 177                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 178                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 179                                 final ElanInstanceCache elanInstanceCache,
 180                                 final ElanInterfaceCache elanInterfaceCache,
 181                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 182                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 183         super(ElanInterface.class, ElanInterfaceManager.class);
 184         this.broker = dataBroker;
 185         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 186         this.idManager = managerService;
 187         this.mdsalManager = mdsalApiManager;
 188         this.interfaceManager = interfaceManager;
 189         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 190         this.neutronVpnManager = neutronVpnManager;
 191         this.elanItmUtils = elanItmUtils;
 192         this.elanEtreeUtils = elanEtreeUtils;
 193         this.elanL2GatewayUtils = elanL2GatewayUtils;
 194         this.elanUtils = elanUtils;
 195         this.jobCoordinator = jobCoordinator;
 196         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 197         this.elanInstanceCache = elanInstanceCache;
 198         this.elanInterfaceCache = elanInterfaceCache;
 199         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 200     }
 201
 202     @Override
 203     @PostConstruct
 204     public void init() {
 205         registerListener();
 206     }
 207
 208     @Override
 209     public void registerListener() {
 210         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 211     }
 212
 213     @Override
 214     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 215         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 216     }
 217
 218     @Override
 219     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 220         String interfaceName = del.getName();
 221         String elanInstanceName = del.getElanInstanceName();
 222         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 223         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 224             elanInterfaces.remove(del);
 225             if (elanInterfaces.isEmpty()) {
 226                 unProcessedElanInterfaces.remove(elanInstanceName);
 227             }
 228         }
 229         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 230         /*
 231          * Handling in case the elan instance is deleted.If the Elan instance is
 232          * deleted, there is no need to explicitly delete the elan interfaces
 233          */
 234         if (elanInfo == null) {
 235             return;
 236         }
 237         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 238         if (interfaceInfo == null && elanInfo.isExternal()) {
 239             // In deleting external network, the underlying ietf Inteface might have been removed
 240             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 241             // from Operational DS instead
 242             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 243         }
 244         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 245                 interfaceName, interfaceInfo, this);
 246         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 247     }
 248
 249     private static class RemoveElanInterfaceHolder {
 250         boolean isLastElanInterface = false;
 251         boolean isLastInterfaceOnDpn = false;
 252         BigInteger dpId = null;
 253     }
 254
 255     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 256     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 257             InterfaceInfo interfaceInfo) {
 258         String elanName = elanInfo.getElanInstanceName();
 259         long elanTag = elanInfo.getElanTag();
 260         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 261         List<ListenableFuture<Void>> futures = new ArrayList<>();
 262         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 263         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 264             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 265             if (elanState == null) {
 266                 return;
 267             }
 268             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 269                 List<String> elanInterfaces = requireNonNullElse(elanState.getElanInterfaces(), emptyList());
 270                 if (elanInterfaces.isEmpty()) {
 271                     holder.isLastElanInterface = true;
 272                 }
 273                 if (interfaceInfo != null) {
 274                     holder.dpId = interfaceInfo.getDpId();
 275                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 276                         interfaceName, elanTag, interfaceTx);
 277                     /*
 278                      * If there are not elan ports, remove the unknown dmac, terminating
 279                      * service table flows, remote/local bc group
 280                      */
 281                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 282                         || dpnInterfaces.getInterfaces().isEmpty()) {
 283                         // No more Elan Interfaces in this DPN
 284                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 285                             holder.dpId);
 286                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 287                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag());
 288                         }
 289                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag());
 290                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 291                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 292                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 293                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 294                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 295                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 296                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 297                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 298                             }
 299                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 300                         }
 301                         holder.isLastInterfaceOnDpn = true;
 302                     } else {
 303                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 304                     }
 305                 }
 306             }));
 307         }));
 308         futures.forEach(ElanUtils::waitForTransactionToComplete);
 309
 310         if (holder.isLastInterfaceOnDpn && holder.dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 311             futures.add(
 312                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 313                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInfo, holder.dpId, confTx))));
 314         }
 315         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 316                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 317         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 318                 ElanConstants.JOB_MAX_RETRIES);
 319
 320         return futures;
 321     }
 322
 323     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 324             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 325             throws ExecutionException, InterruptedException {
 326         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 327         if (etreeLeafTag != null) {
 328             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 329             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 330         }
 331     }
 332
 333     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 334             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 335             throws ExecutionException, InterruptedException {
 336         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 337         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 338     }
 339
 340     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 341             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 342             throws ExecutionException, InterruptedException {
 343         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 344         if (etreeInstance != null) {
 345             BigInteger dpnId = interfaceInfo.getDpId();
 346             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 347             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 348             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 349         }
 350     }
 351
 352     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 353             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 354             throws ExecutionException, InterruptedException {
 355         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 356         if (etreeInstance != null) {
 357             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 358             BigInteger dpnId = interfaceInfo.getDpId();
 359             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 360             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 361             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 362         }
 363     }
 364
 365     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 366             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 367         String elanName = elanInfo.getElanInstanceName();
 368         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 369         if (elanState == null) {
 370             return elanState;
 371         }
 372         List<String> elanInterfaces = requireNonNullElse(elanState.getElanInterfaces(), emptyList());
 373         boolean isRemoved = elanInterfaces.remove(interfaceName);
 374         if (!isRemoved) {
 375             return elanState;
 376         }
 377
 378         if (elanInterfaces.isEmpty()) {
 379             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 380             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 381             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 382         } else {
 383             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 384                     .withKey(new ElanKey(elanName)).build();
 385             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 386         }
 387         return elanState;
 388     }
 389
 390     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 391             throws ReadFailedException {
 392         // removing the ElanInterface from the config data_store if interface is
 393         // not present in Interface config DS
 394         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 395             interfaceName) == null
 396                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 397             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 398         }
 399     }
 400
 401     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 402             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 403         String elanName = elanInfo.getElanInstanceName();
 404         List<ListenableFuture<Void>> futures = new ArrayList<>();
 405         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 406             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 407                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 408                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 409                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 410                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 411                 if (interfaceInfo != null) {
 412                     if (existingElanInterfaceMac.isPresent()) {
 413                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 414                         if (existingMacEntries != null) {
 415                             List<PhysAddress> macAddresses = new ArrayList<>();
 416                             for (MacEntry macEntry : existingMacEntries) {
 417                                 PhysAddress macAddress = macEntry.getMacAddress();
 418                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 419                                         macAddress.getValue(), interfaceName);
 420                                 Optional<MacEntry> macEntryOptional =
 421                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 422                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 423                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 424                                 }
 425                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 426                                 macAddresses.add(macAddress);
 427                             }
 428
 429                             // Removing all those MACs from External Devices belonging
 430                             // to this ELAN
 431                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 432                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 433                             }
 434                         }
 435                     }
 436                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 437                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 438                 } else if (existingElanInterfaceMac.isPresent()) {
 439                     // Interface does not exist in ConfigDS, so lets remove everything
 440                     // about that interface related to Elan
 441                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 442                     if (macEntries != null) {
 443                         for (MacEntry macEntry : macEntries) {
 444                             PhysAddress macAddress = macEntry.getMacAddress();
 445                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 446                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 447                             }
 448                         }
 449                     }
 450                 }
 451                 if (existingElanInterfaceMac.isPresent()) {
 452                     interfaceTx.delete(elanInterfaceId);
 453                 }
 454                 unbindService(interfaceName, flowTx);
 455                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 456             }));
 457         }));
 458         return futures;
 459     }
 460
 461     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 462                                                                          String interfaceName, long elanTag,
 463                                                                          TypedReadWriteTransaction<Operational> tx)
 464             throws ExecutionException, InterruptedException {
 465         synchronized (elanName.intern()) {
 466
 467             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 468             if (dpnInterfaces != null) {
 469                 List<String> interfaceLists = dpnInterfaces.getInterfaces();
 470                 if (interfaceLists != null) {
 471                     interfaceLists.remove(interfaceName);
 472                 }
 473
 474                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 475                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 476                     deleteElanDpnInterface(elanName, dpId, tx);
 477                 } else {
 478                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 479                 }
 480             }
 481             return dpnInterfaces;
 482         }
 483     }
 484
 485     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 486         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 487         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 488             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 489                 BigInteger currentDpId = dpnInterface.getDpId();
 490                 if (!currentDpId.equals(dpId)) {
 491                     for (String elanInterface : requireNonNullElse(dpnInterface.getInterfaces(),
 492                             Collections.<String>emptyList())) {
 493                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 494                         if (macs == null || macs.getMacEntry() == null) {
 495                             continue;
 496                         }
 497                         for (MacEntry mac : macs.getMacEntry()) {
 498                             removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 499                             removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 500                         }
 501                     }
 502                 }
 503             }
 504         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 505     }
 506
 507     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 508             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 509         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 510         if (etreeLeafTag != null) {
 511             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac, confTx);
 512         }
 513     }
 514
 515     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 516             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 517         mdsalManager
 518                 .removeFlow(confTx, dpId,
 519                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 520                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 521                                         mac.getMacAddress().getValue(), elanTag)));
 522     }
 523
 524     /*
 525     * Possible Scenarios for update
 526     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 527         then all {1,2,3,4} should be removed
 528
 529         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 530         then all {1,2,3,4} should be added
 531
 532         c. if orig = {1,2,3,4} updated={2,3,4}
 533         then 1 should be removed
 534
 535         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 536         then we should just add 5
 537
 538         e. if orig = {1,2,3,4} updated={2,3,4,5}
 539         then 1 should be removed , 5 should be added
 540     * */
 541     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 542     @Override
 543     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 544         // updating the static-Mac Entries for the existing elanInterface
 545         String elanName = update.getElanInstanceName();
 546         String interfaceName = update.getName();
 547
 548         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 549         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 550         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 551         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 552
 553         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 554                 deletedEntry.getMacAddress()));
 555
 556         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 557         * Hence add the macentries for the same.*/
 558         for (StaticMacEntries staticMacEntry : updatedEntries) {
 559             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 560                     staticMacEntry.getMacAddress());
 561             ListenableFutures.addErrorLogging(ElanUtils.waitForTransactionToComplete(
 562                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 563                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 564                     if (existingMacEntry.isPresent()) {
 565                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 566                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 567                             tx);
 568                     } else {
 569                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 570                             elanName, interfaceName, staticMacEntry, tx);
 571                     }
 572                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 573         }
 574     }
 575
 576     @Override
 577     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 578         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 579             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 580             String interfaceName = elanInterfaceAdded.getName();
 581             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 582             if (interfaceInfo == null) {
 583                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 584                 return;
 585             }
 586             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 587
 588             if (elanInstance == null) {
 589                 // Add the ElanInstance in the Configuration data-store
 590                 List<String> elanInterfaces = new ArrayList<>();
 591                 elanInterfaces.add(interfaceName);
 592                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 593                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 594                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 595                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 596             }
 597
 598             Long elanTag = elanInstance.getElanTag();
 599             // If elan tag is not updated, then put the elan interface into
 600             // unprocessed entry map and entry. Let entries
 601             // in this map get processed during ELAN update DCN.
 602             if (elanTag == null || elanTag == 0L) {
 603                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 604                 if (elanInterfaces == null) {
 605                     elanInterfaces = new ConcurrentLinkedQueue<>();
 606                 }
 607                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 608                     elanInterfaces.add(elanInterfaceAdded);
 609                 }
 610                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 611                 return;
 612             }
 613             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 614                     interfaceInfo, elanInstance, this);
 615             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 616         }), LOG, "Error processing added ELAN interface");
 617     }
 618
 619     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 620         List<ListenableFuture<Void>> futures = new ArrayList<>();
 621         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 622         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 623             return futures;
 624         }
 625         for (ElanInterface elanInterface : elanInterfaces) {
 626             String interfaceName = elanInterface.getName();
 627             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 628             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 629         }
 630         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 631         return futures;
 632     }
 633
 634     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 635             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 636         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 637                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 638         List<DpnInterfaces> dpnInterfaceLists = null;
 639         if (elanDpnInterfacesList != null) {
 640             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 641         }
 642         if (dpnInterfaceLists == null) {
 643             dpnInterfaceLists = new ArrayList<>();
 644         }
 645         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 646             BigInteger dstDpId = interfaceInfo.getDpId();
 647             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 648                 continue;
 649             }
 650             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 651             for (String remoteIf : remoteElanInterfaces) {
 652                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 653                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 654                 if (elanIfMac == null || remoteInterface == null) {
 655                     continue;
 656                 }
 657                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 658                 if (remoteMacEntries != null) {
 659                     for (MacEntry macEntry : remoteMacEntries) {
 660                         String macAddress = macEntry.getMacAddress().getValue();
 661                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 662                                 dstDpId, elanInstance.getElanInstanceName());
 663                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 664                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 665                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 666                     }
 667                 }
 668             }
 669         }
 670     }
 671
 672     private static class AddElanInterfaceHolder {
 673         private DpnInterfaces dpnInterfaces = null;
 674         private boolean isFirstInterfaceInDpn = false;
 675         private BigInteger dpId;
 676     }
 677
 678     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 679     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 680             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 681         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 682         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 683         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 684
 685         String interfaceName = elanInterface.getName();
 686         String elanInstanceName = elanInterface.getElanInstanceName();
 687
 688         List<ListenableFuture<Void>> futures = new ArrayList<>();
 689         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 690         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 691             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 692             if (elanInfo == null) {
 693                 List<String> elanInterfaces = new ArrayList<>();
 694                 elanInterfaces.add(interfaceName);
 695                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 696                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 697                         operTx)));
 698             } else {
 699                 createElanStateList(elanInstanceName, interfaceName, operTx);
 700             }
 701             // Specific actions to the DPN where the ElanInterface has been added,
 702             // for example, programming the
 703             // External tunnel table if needed or adding the ElanInterface to the
 704             // DpnInterfaces in the operational DS.
 705             holder.dpId = interfaceInfo.getDpId();
 706             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 707                 synchronized (elanInstanceName.intern()) {
 708                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 709                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 710                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 711                     if (ElanUtils.isVlan(elanInstance)) {
 712                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 713                             elanInstanceName, existingElanDpnInterfaces);
 714                     } else {
 715                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 716                     }
 717                     if (holder.isFirstInterfaceInDpn) {
 718                         // ELAN's 1st ElanInterface added to this DPN
 719                         if (!existingElanDpnInterfaces.isPresent()) {
 720                             holder.dpnInterfaces =
 721                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 722                         } else {
 723                             List<String> elanInterfaces =
 724                                 requireNonNullElse(existingElanDpnInterfaces.get().getInterfaces(), emptyList());
 725                             elanInterfaces.add(interfaceName);
 726                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 727                                 elanInterfaces, operTx);
 728                         }
 729                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 730                         // table, but only if Elan has VNI
 731                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 732                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 733                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 734                         }
 735                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 736                             interfaceName);
 737                     } else {
 738                         List<String> elanInterfaces =
 739                             requireNonNullElse(existingElanDpnInterfaces.get().getInterfaces(), emptyList());
 740                         elanInterfaces.add(interfaceName);
 741                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 742                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 743                                 interfaceName);
 744                         }
 745                         holder.dpnInterfaces =
 746                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 747                     }
 748                 }
 749             }
 750
 751             // add code to install Local/Remote BC group, unknow DMAC entry,
 752             // terminating service table flow entry
 753             // call bindservice of interfacemanager to create ingress table flow
 754             // enty.
 755             // Add interface to the ElanInterfaceForwardingEntires Container
 756             createElanInterfaceTablesList(interfaceName, operTx);
 757         }));
 758         futures.forEach(ElanUtils::waitForTransactionToComplete);
 759         futures.add(
 760             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 761                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 762                     holder.isFirstInterfaceInDpn, confTx))));
 763
 764         // add the vlan provider interface to remote BC group for the elan
 765         // for internal vlan networks
 766         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 767             if (interfaceManager.isExternalInterface(interfaceName)) {
 768                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 769                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 770             }
 771         }
 772
 773         if (holder.isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 774             //update the remote-DPNs remoteBC group entry with Tunnels
 775             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance,
 776                 holder.dpId);
 777             futures.add(
 778                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 779                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInstance, holder.dpId, confTx))));
 780         }
 781
 782         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 783         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 784                 elanInterface, interfaceInfo, elanInstance, holder.isFirstInterfaceInDpn, this);
 785         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 786         return futures;
 787     }
 788
 789     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 790     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 791             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 792         String elanInstanceName = elanInstance.getElanInstanceName();
 793         String interfaceName = elanInterface.getName();
 794         List<ListenableFuture<Void>> futures = new ArrayList<>();
 795         BigInteger dpId = interfaceInfo.getDpId();
 796         boolean isInterfaceOperational = isOperational(interfaceInfo);
 797         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 798             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 799                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 800                     isFirstInterfaceInDpn, confTx, operTx);
 801
 802                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 803                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 804
 805                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 806                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 807                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 808                             staticMacEntry.getMacAddress());
 809                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 810                             LogicalDatastoreType.OPERATIONAL, macId);
 811                         if (existingMacEntry.isPresent()) {
 812                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 813                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 814                                 existingMacEntry.get(), operTx);
 815                         } else {
 816                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 817                                 interfaceName, staticMacEntry, operTx);
 818                         }
 819
 820                         if (isInterfaceOperational) {
 821                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 822                             // DPNs
 823                             String macAddress = staticMacEntry.getMacAddress().getValue();
 824                             LOG.info(
 825                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 826                                     + " {}",
 827                                 macAddress, elanInstanceName, interfaceName);
 828                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 829                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 830                         }
 831                     }
 832
 833                     if (isInterfaceOperational) {
 834                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 835                         // on purpose.
 836                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 837                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 838                         }
 839                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 840                             staticMacAddresses);
 841                     }
 842                 }
 843             }));
 844         }));
 845         futures.forEach(ElanUtils::waitForTransactionToComplete);
 846         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 847             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 848             try {
 849                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 850                 if (neutronPort != null) {
 851                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 852                 }
 853             } catch (IllegalArgumentException ex) {
 854                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 855             }
 856         }
 857         return futures;
 858     }
 859
 860     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 861             PhysAddress physAddress) {
 862         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 863         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 864         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 865                 LogicalDatastoreType.OPERATIONAL, macId);
 866
 867         if (!existingMacEntry.isPresent()) {
 868             return;
 869         }
 870
 871         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 872                 .withKey(new MacEntryKey(physAddress)).build();
 873         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 874                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 875     }
 876
 877     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 878             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 879         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 880         if (!existingElanDpnInterfaces.isPresent()) {
 881             return true;
 882         }
 883         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 884             return false;
 885         }
 886         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 887         int dummyInterfaceCount =  0;
 888         List<String> interfaces = requireNonNullElse(dpnInterfaces.getInterfaces(), emptyList());
 889         if (interfaces.contains(routerPortUuid)) {
 890             dummyInterfaceCount++;
 891         }
 892         if (interfaces.contains(elanInstanceName)) {
 893             dummyInterfaceCount++;
 894         }
 895         return interfaces.size() - dummyInterfaceCount == 0;
 896     }
 897
 898     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 899         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 900                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 901     }
 902
 903     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 904             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx,
 905             TypedWriteTransaction<Operational> operTx) {
 906         if (!isOperational(interfaceInfo)) {
 907             return;
 908         }
 909         BigInteger dpId = interfaceInfo.getDpId();
 910         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 911             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 912         }
 913         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 914         if (isFirstInterfaceInDpn) {
 915             // Terminating Service , UnknownDMAC Table.
 916             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 917             // for ELAN Instance is VxLAN
 918             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 919                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 920             }
 921             setupUnknownDMacTable(elanInstance, dpId, confTx);
 922             /*
 923              * Install remote DMAC flow. This is required since this DPN is
 924              * added later to the elan instance and remote DMACs of other
 925              * interfaces in this elan instance are not present in the current
 926              * dpn.
 927              */
 928             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 929                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 930                         elanInstance.getElanInstanceName());
 931                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 932             }
 933         }
 934         // bind the Elan service to the Interface
 935         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 936     }
 937
 938     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 939             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 940         if (!isOperational(interfaceInfo)) {
 941             return;
 942         }
 943         // LocalBroadcast Group creation with elan-Interfaces
 944         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
 945         if (isFirstInterfaceInDpn) {
 946             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 947             BigInteger dpId = interfaceInfo.getDpId();
 948             // RemoteBroadcast Group creation
 949             try {
 950                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 951             } catch (InterruptedException e1) {
 952                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 953             }
 954             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId, confTx);
 955             try {
 956                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 957             } catch (InterruptedException e1) {
 958                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 959             }
 960         }
 961     }
 962
 963     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 964             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 965         int ifTag = interfaceInfo.getInterfaceTag();
 966         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 967                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 968                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 969                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 970                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 971
 972         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
 973
 974         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 975                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 976                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 977                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 978
 979         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
 980     }
 981
 982     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 983             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
 984         int ifTag = interfaceInfo.getInterfaceTag();
 985         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 986             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
 987
 988         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
 989
 990         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 991             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 992             0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 993             getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 994
 995         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
 996     }
 997
 998     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId,
 999             TypedWriteTransaction<Configuration> confTx) {
1000         int elanTag = elanInfo.getElanTag().intValue();
1001         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
1002         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId, confTx);
1003         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1004         if (etreeInstance != null) {
1005             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
1006             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
1007             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId, confTx);
1008         }
1009     }
1010
1011     @SuppressWarnings("checkstyle:IllegalCatch")
1012     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId,
1013             TypedWriteTransaction<Configuration> confTx) {
1014         int bucketId = 0;
1015         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
1016         if (elanDpns != null) {
1017             List<DpnInterfaces> dpnInterfaces = requireNonNullElse(elanDpns.getDpnInterfaces(), emptyList());
1018             for (DpnInterfaces dpnInterface : dpnInterfaces) {
1019                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
1020                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
1021                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
1022                     List<Action> listAction = new ArrayList<>();
1023                     int actionKey = 0;
1024                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1025                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
1026                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1027                     bucketId++;
1028                     for (DpnInterfaces otherFes : dpnInterfaces) {
1029                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
1030                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
1031                             && !otherFes.getInterfaces().isEmpty()) {
1032                             try {
1033                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
1034                                         dpnInterface.getDpId(), otherFes.getDpId(),
1035                                         elanUtils.isOpenstackVniSemanticsEnforced()
1036                                                 ? ElanUtils.getVxlanSegmentationId(elanInfo) : elanTag);
1037                                 if (!remoteListActionInfo.isEmpty()) {
1038                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
1039                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1040                                     bucketId++;
1041                                 }
1042                             } catch (Exception ex) {
1043                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1044                                         + "Logical Group Interface not found between source Dpn - {}, "
1045                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1046                                 return;
1047                             }
1048                         }
1049                     }
1050                     List<Bucket> elanL2GwDevicesBuckets = elanL2GatewayMulticastUtils
1051                             .getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnInterface.getDpId(), bucketId);
1052                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1053
1054                     if (remoteListBucketInfo.isEmpty()) {
1055                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1056                         continue;
1057                     }
1058                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1059                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1060                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1061                     mdsalManager.addGroup(confTx, dpnInterface.getDpId(), group);
1062                 }
1063             }
1064             try {
1065                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1066             } catch (InterruptedException e1) {
1067                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1068             }
1069         }
1070     }
1071
1072     private List<MatchInfo> buildMatchesForVni(Long vni) {
1073         List<MatchInfo> mkMatches = new ArrayList<>();
1074         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1075         mkMatches.add(match);
1076         return mkMatches;
1077     }
1078
1079     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1080         List<InstructionInfo> mkInstructions = new ArrayList<>();
1081         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1082         return mkInstructions;
1083     }
1084
1085     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1086         List<MatchInfo> mkMatches = new ArrayList<>();
1087         // Matching metadata
1088         mkMatches.add(new MatchMetadata(
1089                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1090         return mkMatches;
1091     }
1092
1093     /**
1094      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1095      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1096      *
1097      * @param elanTag
1098      *            elanTag to be written in metadata when flow is selected
1099      * @return the instructions ready to be installed in a flow
1100      */
1101     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1102         List<InstructionInfo> mkInstructions = new ArrayList<>();
1103         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1104                 .getElanMetadataMask()));
1105         /* applicable for EXTERNAL_TUNNEL_TABLE only
1106         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1107         */
1108         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1109         return mkInstructions;
1110     }
1111
1112     // Install DMAC entry on dst DPN
1113     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1114             BigInteger dstDpId) {
1115         String interfaceName = interfaceInfo.getInterfaceName();
1116         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1117         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1118             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1119             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1120                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1121                     for (MacEntry macEntry : macEntries) {
1122                         String macAddress = macEntry.getMacAddress().getValue();
1123                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1124                             interfaceName);
1125                         synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1126                             interfaceInfo.getDpId())) {
1127                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1128                                 macAddress);
1129                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1130                         }
1131                     }
1132                 })));
1133         }
1134         return emptyList();
1135     }
1136
1137     private void createDropBucket(List<Bucket> listBucket) {
1138         List<Action> actionsInfos = new ArrayList<>();
1139         actionsInfos.add(new ActionDrop().buildAction());
1140         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1141                 MDSALUtil.WATCH_GROUP);
1142         listBucket.add(dropBucket);
1143     }
1144
1145     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1146             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1147         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1148         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1149     }
1150
1151     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1152             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1153         List<Bucket> listBucket = new ArrayList<>();
1154         int bucketId = 0;
1155         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1156
1157         List<String> interfaces = new ArrayList<>();
1158         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1159             interfaces = newDpnInterface.getInterfaces();
1160         }
1161         for (String ifName : interfaces) {
1162             // In case if there is a InterfacePort in the cache which is not in
1163             // operational state, skip processing it
1164             InterfaceInfo ifInfo = interfaceManager
1165                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1166             if (!isOperational(ifInfo)) {
1167                 continue;
1168             }
1169
1170             if (!interfaceManager.isExternalInterface(ifName)) {
1171                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1172                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1173                 bucketId++;
1174             }
1175         }
1176
1177         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1178                 MDSALUtil.buildBucketLists(listBucket));
1179         LOG.trace("installing the localBroadCast Group:{}", group);
1180         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1181     }
1182
1183     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1184             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1185         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1186         if (etreeInstance != null) {
1187             List<Bucket> listBucket = new ArrayList<>();
1188             int bucketId = 0;
1189
1190             List<String> interfaces = new ArrayList<>();
1191             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1192                 interfaces = newDpnInterface.getInterfaces();
1193             }
1194             for (String ifName : interfaces) {
1195                 // In case if there is a InterfacePort in the cache which is not
1196                 // in
1197                 // operational state, skip processing it
1198                 InterfaceInfo ifInfo = interfaceManager
1199                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1200                 if (!isOperational(ifInfo)) {
1201                     continue;
1202                 }
1203
1204                 if (!interfaceManager.isExternalInterface(ifName)) {
1205                     // only add root interfaces
1206                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1207                 }
1208             }
1209
1210             if (listBucket.isEmpty()) { // No Buckets
1211                 createDropBucket(listBucket);
1212             }
1213
1214             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1215             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1216             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1217                     MDSALUtil.buildBucketLists(listBucket));
1218             LOG.trace("installing the localBroadCast Group:{}", group);
1219             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1220         }
1221     }
1222
1223     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1224             InterfaceInfo ifInfo) {
1225         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1226         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1227             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1228                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1229             bucketId++;
1230         }
1231         return bucketId;
1232     }
1233
1234     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1235             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1236             throws ExecutionException, InterruptedException {
1237         BigInteger dpnId = interfaceInfo.getDpId();
1238         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1239         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1240         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1241     }
1242
1243     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1244             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1245             throws ExecutionException, InterruptedException {
1246         BigInteger dpnId = interfaceInfo.getDpId();
1247         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1248         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1249         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1250     }
1251
1252     /**
1253      * Installs a flow in the External Tunnel table consisting in translating
1254      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1255      * elanTag that will be used later in the ELANs pipeline.
1256      * @param dpnId the dpn id
1257      */
1258     private void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1259             TypedWriteTransaction<Configuration> confTx) {
1260         long elanTag = elanInfo.getElanTag();
1261         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1262                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1263                 elanInfo.getElanInstanceName(), // flowName
1264                 0, // idleTimeout
1265                 0, // hardTimeout
1266                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1267                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1268                 getInstructionsIntOrExtTunnelTable(elanTag));
1269
1270         mdsalManager.addFlow(confTx, flowEntity);
1271     }
1272
1273     /**
1274      * Removes, from External Tunnel table, the flow that translates from VNI to
1275      * elanTag. Important: ensure this method is only called whenever there is
1276      * no other ElanInterface in the specified DPN
1277      *
1278      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1279      */
1280     private void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1281             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1282         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1283         // last ElanInstance plus
1284         // adding a new one does (almost at the same time) are executed in that
1285         // exact order
1286
1287         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1288         FlowEntity flowEntity = new FlowEntityBuilder()
1289             .setDpnId(dpnId)
1290             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1291             .setFlowId(flowId)
1292             .build();
1293         mdsalManager.removeFlow(confTx, flowEntity);
1294     }
1295
1296     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1297             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1298         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1299         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1300     }
1301
1302     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1303             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1304         List<? extends MatchInfoBase> listMatchInfoBase;
1305         List<InstructionInfo> instructionInfos;
1306         long serviceId;
1307         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1308             serviceId = elanTag;
1309             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1310             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1311         } else {
1312             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1313             listMatchInfoBase = buildMatchesForVni(serviceId);
1314             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1315         }
1316         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1317                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1318                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1319                 instructionInfos);
1320         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1321     }
1322
1323     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1324             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1325         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1326         if (etreeInstance != null) {
1327             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1328         }
1329     }
1330
1331     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId,
1332             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1333         long elanTag = elanInfo.getElanTag();
1334         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1335         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1336         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1337     }
1338
1339     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1340             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1341         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1342         if (etreeLeafTag != null) {
1343             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1344             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1345             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1346         }
1347     }
1348
1349     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1350             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1351         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1352                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1353                 5, elanInfo.getElanInstanceName(), 0, 0,
1354                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1355                 getMatchesForElanTag(elanTag, /* SH flag */false),
1356                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1357
1358         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1359     }
1360
1361     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1362             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1363         // only if ELAN can connect to external network, perform the following
1364
1365         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1366             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1367                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1368                     5, elanInfo.getElanInstanceName(), 0, 0,
1369                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1370                     getMatchesForElanTag(elanTag, /* SH flag */true),
1371                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1372             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1373         }
1374     }
1375
1376
1377     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
1378             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1379             throws ExecutionException, InterruptedException {
1380         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1381                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1382         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1383
1384         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1385             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1386                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1387                     .build();
1388             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1389         }
1390     }
1391
1392     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1393         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1394     }
1395
1396     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1397             TypedWriteTransaction<Configuration> tx) {
1398         if (isStandardElanService(elanInterface)) {
1399             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1400                     elanInterface.getName(), lportTag, tx);
1401         } else { // Etree service
1402             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1403         }
1404     }
1405
1406     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1407             TypedWriteTransaction<Configuration> tx) {
1408         int instructionKey = 0;
1409         List<Instruction> instructions = new ArrayList<>();
1410         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1411                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1412
1413         List<Action> actions = new ArrayList<>();
1414         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1415                 lportTag).buildAction());
1416         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1417                 elanTag).buildAction());
1418         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1419
1420         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1421                 ++instructionKey));
1422
1423         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1424         BoundServices serviceInfo = ElanUtils.getBoundServices(
1425                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1426                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1427         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1428         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1429                 bindServiceId);
1430         if (!existingElanService.isPresent()) {
1431             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1432         }
1433     }
1434
1435     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1436             TypedWriteTransaction<Configuration> tx) {
1437         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1438             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1439                     lportTag, tx);
1440         } else {
1441             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1442             if (etreeInstance == null) {
1443                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1444                         elanInterface.getName(), elanInfo.getElanInstanceName());
1445             } else {
1446                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1447                         elanInterface.getName(), lportTag, tx);
1448             }
1449         }
1450     }
1451
1452     private boolean isStandardElanService(ElanInterface elanInterface) {
1453         return elanInterface.augmentation(EtreeInterface.class) == null;
1454     }
1455
1456     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1457             throws ExecutionException, InterruptedException {
1458         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1459         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1460         if (tx.read(bindServiceId).get().isPresent()) {
1461             tx.delete(bindServiceId);
1462         }
1463     }
1464
1465     private String getFlowRef(long tableId, long elanTag) {
1466         return String.valueOf(tableId) + elanTag;
1467     }
1468
1469     private String getFlowRef(long tableId, long elanTag, String flowName) {
1470         return new StringBuffer().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1471                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1472     }
1473
1474     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1475         return String.valueOf(tableId) + elanTag + shFlag;
1476     }
1477
1478     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1479         List<Action> listAction = new ArrayList<>();
1480         int actionKey = 0;
1481         listAction.add(
1482             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1483         actionKey++;
1484         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1485         return listAction;
1486     }
1487
1488     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1489             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1490         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1491                 .withKey(new DpnInterfacesKey(dpId)).build();
1492         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1493                 CREATE_MISSING_PARENTS);
1494         return dpnInterface;
1495     }
1496
1497     /**
1498      * Delete elan dpn interface from operational DS.
1499      *
1500      * @param elanInstanceName
1501      *            the elan instance name
1502      * @param dpId
1503      *            the dp id
1504      */
1505     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId,
1506             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1507         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1508                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1509         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1510         if (dpnInterfaces.isPresent()) {
1511             tx.delete(dpnInterfacesId);
1512         }
1513     }
1514
1515     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1516             TypedWriteTransaction<Operational> tx) {
1517         List<String> interfaceNames = new ArrayList<>();
1518         interfaceNames.add(interfaceName);
1519         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1520                 .withKey(new DpnInterfacesKey(dpId)).build();
1521         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1522                 CREATE_MISSING_PARENTS);
1523         return dpnInterface;
1524     }
1525
1526     private void createElanInterfaceTablesList(String interfaceName, TypedReadWriteTransaction<Operational> tx)
1527             throws ExecutionException, InterruptedException {
1528         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1529                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1530         Optional<ElanInterfaceMac> interfaceMacTables = tx.read(elanInterfaceMacTables).get();
1531         // Adding new Elan Interface Port to the operational DataStore without
1532         // Static-Mac Entries..
1533         if (!interfaceMacTables.isPresent()) {
1534             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1535                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1536             tx.put(ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1537                     CREATE_MISSING_PARENTS);
1538         }
1539     }
1540
1541     private void createElanStateList(String elanInstanceName, String interfaceName,
1542             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1543         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1544         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1545         // Adding new Elan Interface Port to the operational DataStore without
1546         // Static-Mac Entries..
1547         if (elanInterfaceLists.isPresent()) {
1548             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1549             if (interfaceLists == null) {
1550                 interfaceLists = new ArrayList<>();
1551             }
1552             interfaceLists.add(interfaceName);
1553             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1554                     .withKey(new ElanKey(elanInstanceName)).build();
1555             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1556         }
1557     }
1558
1559     private boolean isOperational(InterfaceInfo interfaceInfo) {
1560         if (interfaceInfo == null) {
1561             return false;
1562         }
1563         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1564     }
1565
1566     @SuppressWarnings("checkstyle:IllegalCatch")
1567     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) {
1568         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1569         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1570                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1571         if (dpnInterfaceLists == null) {
1572             return;
1573         }
1574         List<ElanDpnInterfacesList> elanDpnIf =
1575             requireNonNullElse(dpnInterfaceLists.getElanDpnInterfacesList(), emptyList());
1576         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1577             int cnt = 0;
1578             String elanName = elanDpns.getElanInstanceName();
1579             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1580             if (elanInfo == null) {
1581                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1582                         + "skipping this ELAN for tunnel handling", elanName);
1583                 continue;
1584             }
1585             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1586                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1587                 continue;
1588             }
1589             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1590             if (dpnInterfaces == null) {
1591                 continue;
1592             }
1593             DpnInterfaces dstDpnIf = null;
1594             for (DpnInterfaces dpnIf : dpnInterfaces) {
1595                 BigInteger dpnIfDpId = dpnIf.getDpId();
1596                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1597                     cnt++;
1598                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1599                     cnt++;
1600                     dstDpnIf = dpnIf;
1601                 }
1602             }
1603             if (cnt == 2) {
1604                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1605                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1606                 jobCoordinator.enqueueJob(elanName, () -> {
1607                     // update Remote BC Group
1608                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1609                     try {
1610                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1611                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1612                                 confTx)).get();
1613                     } catch (RuntimeException e) {
1614                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1615                     }
1616                     Set<String> interfaceLists = new HashSet<>();
1617                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1618                     for (String ifName : interfaceLists) {
1619                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1620                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1621                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1622                             if (isOperational(interfaceInfo)) {
1623                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1624                             }
1625                             return emptyList();
1626                         }, ElanConstants.JOB_MAX_RETRIES);
1627                     }
1628                     return emptyList();
1629                 }, ElanConstants.JOB_MAX_RETRIES);
1630             }
1631
1632         }
1633     }
1634
1635     /**
1636      * Handle external tunnel state event.
1637      *
1638      * @param externalTunnel
1639      *            the external tunnel
1640      * @param intrf
1641      *            the interface
1642      */
1643     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1644         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1645             return;
1646         }
1647         // dpId/externalNodeId will be available either in source or destination
1648         // based on the tunnel end point
1649         BigInteger dpId = null;
1650         NodeId externalNodeId = null;
1651         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1652             dpId = new BigInteger(externalTunnel.getSourceDevice());
1653             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1654         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1655             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1656             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1657         }
1658         if (dpId == null || externalNodeId == null) {
1659             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1660             return;
1661         }
1662
1663         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1664         if (dpnInterfaceLists == null) {
1665             return;
1666         }
1667         List<ElanDpnInterfacesList> elanDpnIf =
1668             requireNonNullElse(dpnInterfaceLists.getElanDpnInterfacesList(), emptyList());
1669         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1670             String elanName = elanDpns.getElanInstanceName();
1671             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1672
1673             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1674             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1675                     || dpnInterfaces.getInterfaces().isEmpty()) {
1676                 continue;
1677             }
1678             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1679
1680             final BigInteger finalDpId = dpId;
1681             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1682                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1683                 "Error setting up ELAN BGs");
1684             // install L2gwDevices local macs in dpn.
1685             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1686             // Install dpn macs on external device
1687             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1688                     externalNodeId);
1689         }
1690         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1691     }
1692
1693     /**
1694      * Installs dpn macs in external device. first it checks if the physical
1695      * locator towards this dpn tep is present or not if the physical locator is
1696      * present go ahead and add the ucast macs otherwise update the mcast mac
1697      * entry to include this dpn tep ip and schedule the job to put ucast macs
1698      * once the physical locator is programmed in device
1699      *
1700      * @param elanName
1701      *            the elan name
1702      * @param lstElanInterfaceNames
1703      *            the lst Elan interface names
1704      * @param dpnId
1705      *            the dpn id
1706      * @param externalNodeId
1707      *            the external node id
1708      */
1709     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1710             NodeId externalNodeId) {
1711         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1712                 externalNodeId.getValue());
1713         if (elanL2GwDevice == null) {
1714             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1715             return;
1716         }
1717         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1718         if (dpnTepIp == null) {
1719             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1720                     externalNodeId);
1721             return;
1722         }
1723
1724         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1725         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1726                 LogicalDatastoreType.OPERATIONAL);
1727         boolean phyLocAlreadyExists =
1728                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1729                 dpnTepIp);
1730         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1731                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1732         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1733
1734         if (phyLocAlreadyExists) {
1735             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1736             return;
1737         }
1738         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1739         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1740     }
1741
1742     /**
1743      * Validate external tunnel state event.
1744      *
1745      * @param externalTunnel
1746      *            the external tunnel
1747      * @param intrf
1748      *            the intrf
1749      * @return true, if successful
1750      */
1751     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1752         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1753             String srcDevice = externalTunnel.getDestinationDevice();
1754             String destDevice = externalTunnel.getSourceDevice();
1755             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1756                     LogicalDatastoreType.CONFIGURATION);
1757             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1758                     otherEndPointExtTunnel);
1759             if (otherEndPointExtTunnel != null) {
1760                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1761                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1762                 if (otherEndPointInterfaceOperational) {
1763                     return true;
1764                 } else {
1765                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1766                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1767                 }
1768             }
1769         }
1770         return false;
1771     }
1772
1773     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1774         List<MatchInfo> mkMatches = new ArrayList<>();
1775         // Matching metadata
1776         mkMatches.add(
1777                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1778         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1779         return mkMatches;
1780     }
1781
1782     @Override
1783     protected ElanInterfaceManager getDataTreeChangeListener() {
1784         return this;
1785     }
1786
1787     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1788                                              BigInteger dpId) {
1789         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1790         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1791             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1792                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1793         try {
1794             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1795         } catch (InterruptedException e) {
1796             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1797         }
1798     }
1799 }